uk_bin_collection 0.74.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. uk_bin_collection/README.rst +0 -0
  2. uk_bin_collection/tests/council_feature_input_parity.py +79 -0
  3. uk_bin_collection/tests/features/environment.py +7 -0
  4. uk_bin_collection/tests/features/validate_council_outputs.feature +767 -0
  5. uk_bin_collection/tests/input.json +1077 -0
  6. uk_bin_collection/tests/output.schema +41 -0
  7. uk_bin_collection/tests/step_defs/step_helpers/file_handler.py +46 -0
  8. uk_bin_collection/tests/step_defs/test_validate_council.py +87 -0
  9. uk_bin_collection/tests/test_collect_data.py +104 -0
  10. uk_bin_collection/tests/test_common_functions.py +342 -0
  11. uk_bin_collection/uk_bin_collection/collect_data.py +133 -0
  12. uk_bin_collection/uk_bin_collection/common.py +292 -0
  13. uk_bin_collection/uk_bin_collection/councils/AdurAndWorthingCouncils.py +43 -0
  14. uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py +97 -0
  15. uk_bin_collection/uk_bin_collection/councils/AylesburyValeCouncil.py +69 -0
  16. uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py +51 -0
  17. uk_bin_collection/uk_bin_collection/councils/BarnetCouncil.py +180 -0
  18. uk_bin_collection/uk_bin_collection/councils/BarnsleyMBCouncil.py +109 -0
  19. uk_bin_collection/uk_bin_collection/councils/BasingstokeCouncil.py +72 -0
  20. uk_bin_collection/uk_bin_collection/councils/BathAndNorthEastSomersetCouncil.py +100 -0
  21. uk_bin_collection/uk_bin_collection/councils/BedfordBoroughCouncil.py +49 -0
  22. uk_bin_collection/uk_bin_collection/councils/BedfordshireCouncil.py +70 -0
  23. uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py +147 -0
  24. uk_bin_collection/uk_bin_collection/councils/BirminghamCityCouncil.py +119 -0
  25. uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py +105 -0
  26. uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py +104 -0
  27. uk_bin_collection/uk_bin_collection/councils/BradfordMDC.py +103 -0
  28. uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py +137 -0
  29. uk_bin_collection/uk_bin_collection/councils/BristolCityCouncil.py +141 -0
  30. uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py +115 -0
  31. uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py +107 -0
  32. uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py +95 -0
  33. uk_bin_collection/uk_bin_collection/councils/BuryCouncil.py +65 -0
  34. uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py +123 -0
  35. uk_bin_collection/uk_bin_collection/councils/CannockChaseDistrictCouncil.py +65 -0
  36. uk_bin_collection/uk_bin_collection/councils/CardiffCouncil.py +172 -0
  37. uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py +96 -0
  38. uk_bin_collection/uk_bin_collection/councils/CharnwoodBoroughCouncil.py +54 -0
  39. uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py +127 -0
  40. uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py +32 -0
  41. uk_bin_collection/uk_bin_collection/councils/CheshireWestAndChesterCouncil.py +125 -0
  42. uk_bin_collection/uk_bin_collection/councils/ChorleyCouncil.py +134 -0
  43. uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py +27 -0
  44. uk_bin_collection/uk_bin_collection/councils/CrawleyBoroughCouncil.py +61 -0
  45. uk_bin_collection/uk_bin_collection/councils/CroydonCouncil.py +291 -0
  46. uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py +100 -0
  47. uk_bin_collection/uk_bin_collection/councils/DoncasterCouncil.py +77 -0
  48. uk_bin_collection/uk_bin_collection/councils/DorsetCouncil.py +58 -0
  49. uk_bin_collection/uk_bin_collection/councils/DoverDistrictCouncil.py +41 -0
  50. uk_bin_collection/uk_bin_collection/councils/DurhamCouncil.py +49 -0
  51. uk_bin_collection/uk_bin_collection/councils/EastCambridgeshireCouncil.py +44 -0
  52. uk_bin_collection/uk_bin_collection/councils/EastDevonDC.py +74 -0
  53. uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py +108 -0
  54. uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py +142 -0
  55. uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py +112 -0
  56. uk_bin_collection/uk_bin_collection/councils/EastleighBoroughCouncil.py +70 -0
  57. uk_bin_collection/uk_bin_collection/councils/EnvironmentFirst.py +48 -0
  58. uk_bin_collection/uk_bin_collection/councils/ErewashBoroughCouncil.py +61 -0
  59. uk_bin_collection/uk_bin_collection/councils/FenlandDistrictCouncil.py +65 -0
  60. uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py +113 -0
  61. uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py +118 -0
  62. uk_bin_collection/uk_bin_collection/councils/GedlingBoroughCouncil.py +1580 -0
  63. uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py +55 -0
  64. uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py +150 -0
  65. uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py +142 -0
  66. uk_bin_collection/uk_bin_collection/councils/HaringeyCouncil.py +59 -0
  67. uk_bin_collection/uk_bin_collection/councils/HarrogateBoroughCouncil.py +63 -0
  68. uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py +134 -0
  69. uk_bin_collection/uk_bin_collection/councils/HullCityCouncil.py +48 -0
  70. uk_bin_collection/uk_bin_collection/councils/HuntingdonDistrictCouncil.py +44 -0
  71. uk_bin_collection/uk_bin_collection/councils/KingstonUponThamesCouncil.py +84 -0
  72. uk_bin_collection/uk_bin_collection/councils/KirkleesCouncil.py +130 -0
  73. uk_bin_collection/uk_bin_collection/councils/KnowsleyMBCouncil.py +139 -0
  74. uk_bin_collection/uk_bin_collection/councils/LancasterCityCouncil.py +71 -0
  75. uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py +137 -0
  76. uk_bin_collection/uk_bin_collection/councils/LisburnCastlereaghCityCouncil.py +101 -0
  77. uk_bin_collection/uk_bin_collection/councils/LiverpoolCityCouncil.py +65 -0
  78. uk_bin_collection/uk_bin_collection/councils/LondonBoroughHounslow.py +82 -0
  79. uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py +161 -0
  80. uk_bin_collection/uk_bin_collection/councils/MaldonDistrictCouncil.py +52 -0
  81. uk_bin_collection/uk_bin_collection/councils/MalvernHillsDC.py +57 -0
  82. uk_bin_collection/uk_bin_collection/councils/ManchesterCityCouncil.py +106 -0
  83. uk_bin_collection/uk_bin_collection/councils/MansfieldDistrictCouncil.py +38 -0
  84. uk_bin_collection/uk_bin_collection/councils/MertonCouncil.py +58 -0
  85. uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py +128 -0
  86. uk_bin_collection/uk_bin_collection/councils/MidSussexDistrictCouncil.py +80 -0
  87. uk_bin_collection/uk_bin_collection/councils/MiltonKeynesCityCouncil.py +54 -0
  88. uk_bin_collection/uk_bin_collection/councils/MoleValleyDistrictCouncil.py +98 -0
  89. uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py +139 -0
  90. uk_bin_collection/uk_bin_collection/councils/NewarkAndSherwoodDC.py +52 -0
  91. uk_bin_collection/uk_bin_collection/councils/NewcastleCityCouncil.py +57 -0
  92. uk_bin_collection/uk_bin_collection/councils/NewhamCouncil.py +58 -0
  93. uk_bin_collection/uk_bin_collection/councils/NewportCityCouncil.py +203 -0
  94. uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py +115 -0
  95. uk_bin_collection/uk_bin_collection/councils/NorthEastLincs.py +53 -0
  96. uk_bin_collection/uk_bin_collection/councils/NorthKestevenDistrictCouncil.py +45 -0
  97. uk_bin_collection/uk_bin_collection/councils/NorthLanarkshireCouncil.py +46 -0
  98. uk_bin_collection/uk_bin_collection/councils/NorthLincolnshireCouncil.py +58 -0
  99. uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py +108 -0
  100. uk_bin_collection/uk_bin_collection/councils/NorthNorthamptonshireCouncil.py +72 -0
  101. uk_bin_collection/uk_bin_collection/councils/NorthSomersetCouncil.py +76 -0
  102. uk_bin_collection/uk_bin_collection/councils/NorthTynesideCouncil.py +220 -0
  103. uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py +114 -0
  104. uk_bin_collection/uk_bin_collection/councils/NorthYorkshire.py +58 -0
  105. uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py +123 -0
  106. uk_bin_collection/uk_bin_collection/councils/NottinghamCityCouncil.py +36 -0
  107. uk_bin_collection/uk_bin_collection/councils/OldhamCouncil.py +51 -0
  108. uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py +131 -0
  109. uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py +97 -0
  110. uk_bin_collection/uk_bin_collection/councils/ReadingBoroughCouncil.py +30 -0
  111. uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py +81 -0
  112. uk_bin_collection/uk_bin_collection/councils/RenfrewshireCouncil.py +135 -0
  113. uk_bin_collection/uk_bin_collection/councils/RhonddaCynonTaffCouncil.py +80 -0
  114. uk_bin_collection/uk_bin_collection/councils/RochdaleCouncil.py +69 -0
  115. uk_bin_collection/uk_bin_collection/councils/RochfordCouncil.py +60 -0
  116. uk_bin_collection/uk_bin_collection/councils/RugbyBoroughCouncil.py +93 -0
  117. uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py +100 -0
  118. uk_bin_collection/uk_bin_collection/councils/RushmoorCouncil.py +81 -0
  119. uk_bin_collection/uk_bin_collection/councils/SalfordCityCouncil.py +70 -0
  120. uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py +106 -0
  121. uk_bin_collection/uk_bin_collection/councils/SheffieldCityCouncil.py +54 -0
  122. uk_bin_collection/uk_bin_collection/councils/ShropshireCouncil.py +45 -0
  123. uk_bin_collection/uk_bin_collection/councils/SolihullCouncil.py +48 -0
  124. uk_bin_collection/uk_bin_collection/councils/SomersetCouncil.py +203 -0
  125. uk_bin_collection/uk_bin_collection/councils/SouthAyrshireCouncil.py +73 -0
  126. uk_bin_collection/uk_bin_collection/councils/SouthCambridgeshireCouncil.py +65 -0
  127. uk_bin_collection/uk_bin_collection/councils/SouthGloucestershireCouncil.py +74 -0
  128. uk_bin_collection/uk_bin_collection/councils/SouthLanarkshireCouncil.py +78 -0
  129. uk_bin_collection/uk_bin_collection/councils/SouthNorfolkCouncil.py +91 -0
  130. uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py +93 -0
  131. uk_bin_collection/uk_bin_collection/councils/SouthTynesideCouncil.py +98 -0
  132. uk_bin_collection/uk_bin_collection/councils/StAlbansCityAndDistrictCouncil.py +43 -0
  133. uk_bin_collection/uk_bin_collection/councils/StHelensBC.py +56 -0
  134. uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py +112 -0
  135. uk_bin_collection/uk_bin_collection/councils/StockportBoroughCouncil.py +39 -0
  136. uk_bin_collection/uk_bin_collection/councils/StokeOnTrentCityCouncil.py +79 -0
  137. uk_bin_collection/uk_bin_collection/councils/StratfordUponAvonCouncil.py +94 -0
  138. uk_bin_collection/uk_bin_collection/councils/SunderlandCityCouncil.py +100 -0
  139. uk_bin_collection/uk_bin_collection/councils/SwaleBoroughCouncil.py +52 -0
  140. uk_bin_collection/uk_bin_collection/councils/TamesideMBCouncil.py +62 -0
  141. uk_bin_collection/uk_bin_collection/councils/TandridgeDistrictCouncil.py +60 -0
  142. uk_bin_collection/uk_bin_collection/councils/TelfordAndWrekinCouncil.py +50 -0
  143. uk_bin_collection/uk_bin_collection/councils/TestValleyBoroughCouncil.py +203 -0
  144. uk_bin_collection/uk_bin_collection/councils/TonbridgeAndMallingBC.py +101 -0
  145. uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py +51 -0
  146. uk_bin_collection/uk_bin_collection/councils/TorridgeDistrictCouncil.py +154 -0
  147. uk_bin_collection/uk_bin_collection/councils/ValeofGlamorganCouncil.py +119 -0
  148. uk_bin_collection/uk_bin_collection/councils/ValeofWhiteHorseCouncil.py +103 -0
  149. uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py +89 -0
  150. uk_bin_collection/uk_bin_collection/councils/WarwickDistrictCouncil.py +34 -0
  151. uk_bin_collection/uk_bin_collection/councils/WaverleyBoroughCouncil.py +119 -0
  152. uk_bin_collection/uk_bin_collection/councils/WealdenDistrictCouncil.py +86 -0
  153. uk_bin_collection/uk_bin_collection/councils/WelhatCouncil.py +73 -0
  154. uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py +134 -0
  155. uk_bin_collection/uk_bin_collection/councils/WestLindseyDistrictCouncil.py +118 -0
  156. uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py +103 -0
  157. uk_bin_collection/uk_bin_collection/councils/WestNorthamptonshireCouncil.py +34 -0
  158. uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py +64 -0
  159. uk_bin_collection/uk_bin_collection/councils/WiganBoroughCouncil.py +97 -0
  160. uk_bin_collection/uk_bin_collection/councils/WiltshireCouncil.py +135 -0
  161. uk_bin_collection/uk_bin_collection/councils/WindsorAndMaidenheadCouncil.py +134 -0
  162. uk_bin_collection/uk_bin_collection/councils/WokingBoroughCouncil.py +114 -0
  163. uk_bin_collection/uk_bin_collection/councils/WyreCouncil.py +89 -0
  164. uk_bin_collection/uk_bin_collection/councils/YorkCouncil.py +45 -0
  165. uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py +33 -0
  166. uk_bin_collection/uk_bin_collection/get_bin_data.py +165 -0
  167. uk_bin_collection-0.74.0.dist-info/LICENSE +21 -0
  168. uk_bin_collection-0.74.0.dist-info/METADATA +247 -0
  169. uk_bin_collection-0.74.0.dist-info/RECORD +171 -0
  170. uk_bin_collection-0.74.0.dist-info/WHEEL +4 -0
  171. uk_bin_collection-0.74.0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,135 @@
1
+ from bs4 import BeautifulSoup
2
+ from selenium.webdriver.common.by import By
3
+ from selenium.webdriver.support import expected_conditions as EC
4
+ from selenium.webdriver.support.wait import WebDriverWait
5
+
6
+ from uk_bin_collection.uk_bin_collection.common import *
7
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
8
+
9
+
10
+ # import the wonderful Beautiful Soup and the URL grabber
11
+ class CouncilClass(AbstractGetBinDataClass):
12
+ """
13
+ Concrete classes have to implement all abstract operations of the
14
+ base class. They can also override some operations with a default
15
+ implementation.
16
+ """
17
+
18
+ def parse_data(self, page: str, **kwargs) -> dict:
19
+ driver = None
20
+ try:
21
+ data = {"bins": []}
22
+ user_paon = kwargs.get("paon")
23
+ user_postcode = kwargs.get("postcode")
24
+ web_driver = kwargs.get("web_driver")
25
+ headless = kwargs.get("headless")
26
+ check_paon(user_paon)
27
+ check_postcode(user_postcode)
28
+
29
+ # Create Selenium webdriver
30
+ driver = create_webdriver(web_driver, headless)
31
+ driver.get(
32
+ "https://www.renfrewshire.gov.uk/article/2320/Check-your-bin-collection-day"
33
+ )
34
+
35
+ accept_button = WebDriverWait(driver, timeout=30).until(
36
+ EC.element_to_be_clickable((By.ID, "ccc-notify-accept"))
37
+ )
38
+ accept_button.click()
39
+
40
+ # Wait for the postcode field to appear then populate it
41
+ inputElement_postcode = WebDriverWait(driver, 30).until(
42
+ EC.presence_of_element_located(
43
+ (By.ID, "RENFREWSHIREBINCOLLECTIONS_PAGE1_ADDRESSLOOKUPPOSTCODE")
44
+ )
45
+ )
46
+ inputElement_postcode.send_keys(user_postcode)
47
+
48
+ # Click search button
49
+ findAddress = WebDriverWait(driver, 10).until(
50
+ EC.presence_of_element_located(
51
+ (By.ID, "RENFREWSHIREBINCOLLECTIONS_PAGE1_ADDRESSLOOKUPSEARCH")
52
+ )
53
+ )
54
+ findAddress.click()
55
+
56
+ # Wait for the 'Select address' dropdown to appear and select option matching the house name/number
57
+ WebDriverWait(driver, 10).until(
58
+ EC.element_to_be_clickable(
59
+ (
60
+ By.XPATH,
61
+ "//select[@id='RENFREWSHIREBINCOLLECTIONS_PAGE1_ADDRESSLOOKUPADDRESS']//option[contains(., '"
62
+ + user_paon
63
+ + "')]",
64
+ )
65
+ )
66
+ ).click()
67
+
68
+ # Wait for the collections table to appear
69
+ WebDriverWait(driver, 10).until(
70
+ EC.presence_of_element_located(
71
+ (By.ID, "RENFREWSHIREBINCOLLECTIONS_PAGE1_COLLECTIONDETAILS")
72
+ )
73
+ )
74
+
75
+ soup = BeautifulSoup(driver.page_source, features="html.parser")
76
+
77
+ next_collection_div = soup.find(
78
+ "div", {"class": "collection collection--next"}
79
+ )
80
+
81
+ next_collection_date = datetime.strptime(
82
+ next_collection_div.find("p", {"class": "collection__date"})
83
+ .get_text()
84
+ .strip(),
85
+ "%A %d %B %Y",
86
+ )
87
+
88
+ next_collection_bin = next_collection_div.findAll(
89
+ "p", {"class": "bins__name"}
90
+ )
91
+
92
+ for row in next_collection_bin:
93
+ dict_data = {
94
+ "type": row.get_text().strip(),
95
+ "collectionDate": next_collection_date.strftime("%d/%m/%Y"),
96
+ }
97
+ data["bins"].append(dict_data)
98
+
99
+ future_collection_div = soup.find(
100
+ "div", {"class": "collection collection--future"}
101
+ )
102
+
103
+ future_collection_row = future_collection_div.findAll(
104
+ "div", {"class": "collection__row"}
105
+ )
106
+
107
+ for collection_row in future_collection_row:
108
+ future_collection_date = datetime.strptime(
109
+ collection_row.find("p", {"class": "collection__date"})
110
+ .get_text()
111
+ .strip(),
112
+ "%A %d %B %Y",
113
+ )
114
+ future_collection_bin = collection_row.findAll(
115
+ "p", {"class": "bins__name"}
116
+ )
117
+
118
+ for row in future_collection_bin:
119
+ dict_data = {
120
+ "type": row.get_text().strip(),
121
+ "collectionDate": future_collection_date.strftime("%d/%m/%Y"),
122
+ }
123
+
124
+ data["bins"].append(dict_data)
125
+
126
+ except Exception as e:
127
+ # Here you can log the exception if needed
128
+ print(f"An error occurred: {e}")
129
+ # Optionally, re-raise the exception if you want it to propagate
130
+ raise
131
+ finally:
132
+ # This block ensures that the driver is closed regardless of an exception
133
+ if driver:
134
+ driver.quit()
135
+ return data
@@ -0,0 +1,80 @@
1
+ from bs4 import BeautifulSoup
2
+
3
+ from uk_bin_collection.uk_bin_collection.common import *
4
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
5
+
6
+
7
+ class CouncilClass(AbstractGetBinDataClass):
8
+ """
9
+ Concrete classes have to implement all abstract operations of the
10
+ base class. They can also override some operations with a default
11
+ implementation.
12
+ """
13
+
14
+ def parse_data(self, page: str, **kwargs) -> dict:
15
+ data = {"bins": []}
16
+
17
+ # Get UPRN from kwargs
18
+ user_uprn = kwargs.get("uprn")
19
+ check_uprn(user_uprn)
20
+
21
+ # Construct headers and set passed UPRN as a param for request
22
+ headers = {
23
+ "authority": "www.rctcbc.gov.uk",
24
+ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
25
+ "accept-language": "en-GB,en;q=0.9",
26
+ "cache-control": "no-cache",
27
+ # 'cookie': 'ASP.NET_SessionId=2b4gaaydt1rlu5pccgrpzamm',
28
+ "pragma": "no-cache",
29
+ "referer": "https://www.rctcbc.gov.uk/EN/Resident/RecyclingandWaste/RecyclingandWasteCollectionDays.aspx?PropertyNumber=&Postcode=CF72%209JD",
30
+ "sec-ch-ua": '"Chromium";v="116", "Not)A;Brand";v="24", "Opera GX";v="102"',
31
+ "sec-ch-ua-mobile": "?0",
32
+ "sec-ch-ua-platform": '"Windows"',
33
+ "sec-fetch-dest": "document",
34
+ "sec-fetch-mode": "navigate",
35
+ "sec-fetch-site": "same-origin",
36
+ "sec-fetch-user": "?1",
37
+ "upgrade-insecure-requests": "1",
38
+ "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.188 Safari/537.36",
39
+ }
40
+ params = {
41
+ "uprn": user_uprn,
42
+ }
43
+
44
+ response = requests.get(
45
+ "https://www.rctcbc.gov.uk/EN/Resident/RecyclingandWaste/RecyclingandWasteCollectionDays.aspx",
46
+ params=params,
47
+ headers=headers,
48
+ )
49
+
50
+ # Throw an error if response is not HTTP200
51
+ if response.status_code != 200:
52
+ raise SystemError(
53
+ "Response status was not 200: Please raise an issue on GitHub!"
54
+ )
55
+
56
+ # Parse response page and get all table rows
57
+ soup = BeautifulSoup(response.text, features="html.parser")
58
+ soup.prettify()
59
+ table_rows = soup.find("table", class_=("waste-table")).find_all("tr")
60
+
61
+ # Find the bin type and parse the date for each row, then add to dict. First row will be just headings,
62
+ # so skip it
63
+ for row in table_rows:
64
+ if len(row.contents) > 3:
65
+ continue
66
+ bin_type = row.contents[1].text.strip()
67
+ bin_date_text = (
68
+ row.contents[2].text.split(",")[1].strip().replace(" of ", "")
69
+ )
70
+ bin_date = datetime.strptime(
71
+ remove_ordinal_indicator_from_date_string(bin_date_text), "%A %d %B %Y"
72
+ ).strftime(date_format)
73
+
74
+ dict_data = {
75
+ "type": bin_type,
76
+ "collectionDate": bin_date,
77
+ }
78
+ data["bins"].append(dict_data)
79
+
80
+ return data
@@ -0,0 +1,69 @@
1
+ from bs4 import BeautifulSoup
2
+ from uk_bin_collection.uk_bin_collection.common import *
3
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
4
+
5
+
6
+ # import the wonderful Beautiful Soup and the URL grabber
7
+ class CouncilClass(AbstractGetBinDataClass):
8
+ """
9
+ Concrete classes have to implement all abstract operations of the
10
+ base class. They can also override some operations with a default
11
+ implementation.
12
+ """
13
+
14
+ def parse_data(self, page: str, **kwargs) -> dict:
15
+ api_url = "https://webforms.rochdale.gov.uk/BinCalendar"
16
+ user_postcode = kwargs.get("postcode")
17
+ user_uprn = kwargs.get("uprn")
18
+
19
+ # Check the postcode and UPRN are valid
20
+ check_postcode(user_postcode)
21
+ check_uprn(user_uprn)
22
+
23
+ # Create the form data
24
+ form_data = {
25
+ "PostCode": user_postcode,
26
+ "SelectedUprn": user_uprn,
27
+ "Step": 2,
28
+ }
29
+
30
+ # Make a request to the API
31
+ requests.packages.urllib3.disable_warnings()
32
+ response = requests.post(api_url, data=form_data)
33
+
34
+ # Make a BS4 object
35
+ soup = BeautifulSoup(response.text, features="html.parser")
36
+ soup.prettify()
37
+
38
+ data = {"bins": []}
39
+
40
+ # Get the table element and rows
41
+ table_element = soup.find("table", {"id": "tblCollectionDetails"})
42
+ table_rows = table_element.find_all_next("tr")
43
+
44
+ row_index = 0
45
+ for row in table_rows:
46
+ if row_index < 1:
47
+ row_index += 1
48
+ continue
49
+ else:
50
+ # Get the date from the th element
51
+ date = datetime.strptime(
52
+ row.find("th").get_text().strip(), "%A %d %B %Y"
53
+ ).strftime(date_format)
54
+
55
+ # Get the bin types from the td elements and filter out the empty ones
56
+ bin_types = filter(lambda td: td.find("img"), row.find_all("td"))
57
+
58
+ # Convert the bin types to a list
59
+ bin_types_list = list(bin_types)
60
+
61
+ # Append the bin type and date to the data dict
62
+ for td in bin_types_list:
63
+ img = td.find("img")
64
+ bin_type_text = img["alt"]
65
+ data["bins"].append({"type": bin_type_text, "collectionDate": date})
66
+
67
+ row_index += 1
68
+
69
+ return data
@@ -0,0 +1,60 @@
1
+ from bs4 import BeautifulSoup
2
+ from uk_bin_collection.uk_bin_collection.common import *
3
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
4
+ from dateutil.relativedelta import relativedelta
5
+ from datetime import timedelta
6
+
7
+
8
+ # import the wonderful Beautiful Soup and the URL grabber
9
+ class CouncilClass(AbstractGetBinDataClass):
10
+ """
11
+ Concrete classes have to implement all abstract operations of the
12
+ base class. They can also override some operations with a default
13
+ implementation.
14
+ """
15
+
16
+ def parse_data(self, page: str, **kwargs) -> dict:
17
+ data = {"bins": []}
18
+
19
+ # response = requests.get('https://www.rochford.gov.uk/online-bin-collections-calendar', headers=headers)
20
+ soup = BeautifulSoup(page.text, features="html.parser")
21
+ soup.prettify()
22
+ year = soup.find_all("table", {"class": "responsive-enabled govuk-table"})
23
+
24
+ current_month = datetime.now().strftime("%B %Y")
25
+ next_month = (datetime.now() + relativedelta(months=1, day=1)).strftime("%B %Y")
26
+
27
+ for month in year:
28
+ heading = (
29
+ month.find("th", {"class": "govuk-table__header"}).get_text().strip()
30
+ )
31
+ if heading == current_month or heading == next_month:
32
+ for week in month.find("tbody").find_all(
33
+ "tr", {"class": "govuk-table__row"}
34
+ ):
35
+ week_text = week.get_text().strip().split("\n")
36
+ collection_date = datetime.strptime(
37
+ remove_ordinal_indicator_from_date_string(
38
+ week_text[0].split(" - ")[0]
39
+ ),
40
+ "%A %d %B",
41
+ )
42
+ next_collection = collection_date.replace(year=datetime.now().year)
43
+ if datetime.now().month == 12 and next_collection.month == 1:
44
+ next_collection = next_collection + relativedelta(years=1)
45
+ bin_type = (
46
+ week_text[1]
47
+ .replace("collection week", "bin")
48
+ .strip()
49
+ .capitalize()
50
+ )
51
+ if next_collection.date() >= (datetime.now().date() - timedelta(6)):
52
+ dict_data = {
53
+ "type": bin_type,
54
+ "collectionDate": next_collection.strftime(date_format),
55
+ }
56
+ data["bins"].append(dict_data)
57
+ else:
58
+ continue
59
+
60
+ return data
@@ -0,0 +1,93 @@
1
+ from bs4 import BeautifulSoup
2
+ from uk_bin_collection.uk_bin_collection.common import *
3
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
4
+
5
+
6
+ # import the wonderful Beautiful Soup and the URL grabber
7
+ class CouncilClass(AbstractGetBinDataClass):
8
+ """
9
+ Concrete classes have to implement all abstract operations of the
10
+ base class. They can also override some operations with a default
11
+ implementation.
12
+ """
13
+
14
+ def parse_data(self, page: str, **kwargs) -> dict:
15
+ data = {"bins": []}
16
+ bin_types = {
17
+ "240L RUBBISH BIN": "Black bin",
18
+ "240L GARDEN BIN": "Green bin",
19
+ "180L RECYCLING BIN": "Blue lid bin",
20
+ }
21
+ collections = []
22
+
23
+ user_postcode = kwargs.get("postcode")
24
+ user_uprn = kwargs.get("uprn")
25
+
26
+ check_uprn(user_uprn)
27
+ check_postcode(user_postcode)
28
+
29
+ headers = {
30
+ "authority": "www.rugby.gov.uk",
31
+ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
32
+ "accept-language": "en-GB,en;q=0.9",
33
+ "cache-control": "no-cache",
34
+ "content-type": "application/x-www-form-urlencoded",
35
+ # 'cookie': 'JSESSIONID=7E90CAB54B649C3DCC7F6B5DA0897C63; COOKIE_SUPPORT=true; GUEST_LANGUAGE_ID=en_GB; AWSELB=D941E98916B5759862ED6C39DA9FB3FD9880491851D200C98112ABEC3223D52B19A2A2C6B37A89D3650D44FA5728FCAFEDE7BB2592D948FFF9C7B18D76C41AF02C308B0F3A2DE17F1585E9959BCE68CC83BC3AC753; CookieControl={"necessaryCookies":[],"optionalCookies":{},"statement":{},"consentDate":1701710876715,"consentExpiry":90,"interactedWith":true,"user":"8FED8810-3C3E-4D50-A9DC-42655030B3B1"}',
36
+ "origin": "https://www.rugby.gov.uk",
37
+ "pragma": "no-cache",
38
+ "referer": "https://www.rugby.gov.uk/check-your-next-bin-day",
39
+ "sec-ch-ua": '"Chromium";v="118", "Opera GX";v="104", "Not=A?Brand";v="99"',
40
+ "sec-ch-ua-mobile": "?0",
41
+ "sec-ch-ua-platform": '"Windows"',
42
+ "sec-fetch-dest": "document",
43
+ "sec-fetch-mode": "navigate",
44
+ "sec-fetch-site": "same-origin",
45
+ "sec-fetch-user": "?1",
46
+ "upgrade-insecure-requests": "1",
47
+ "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.5993.118 Safari/537.36",
48
+ }
49
+ params = {
50
+ "p_p_id": "com_placecube_digitalplace_local_waste_portlet_CollectionDayFinderPortlet",
51
+ "p_p_lifecycle": "0",
52
+ "p_p_state": "normal",
53
+ "p_p_mode": "view",
54
+ "_com_placecube_digitalplace_local_waste_portlet_CollectionDayFinderPortlet_mvcRenderCommandName": "/collection_day_finder/get_days",
55
+ }
56
+ data = {
57
+ "_com_placecube_digitalplace_local_waste_portlet_CollectionDayFinderPortlet_formDate": f"{datetime.now().timestamp().__floor__()}",
58
+ "_com_placecube_digitalplace_local_waste_portlet_CollectionDayFinderPortlet_postcode": f"{user_postcode}",
59
+ "_com_placecube_digitalplace_local_waste_portlet_CollectionDayFinderPortlet_uprn": f"{user_uprn}",
60
+ }
61
+
62
+ response = requests.post(
63
+ "https://www.rugby.gov.uk/check-your-next-bin-day",
64
+ params=params,
65
+ headers=headers,
66
+ data=data,
67
+ )
68
+
69
+ soup = BeautifulSoup(response.text, features="html.parser")
70
+ soup.prettify()
71
+
72
+ table_rows = soup.find("table", {"class": "table"}).find("tbody").find_all("tr")
73
+
74
+ for row in table_rows:
75
+ row_text = row.text.strip().split("\n")
76
+ bin_type = bin_types.get(row_text[0])
77
+ collections.append(
78
+ (bin_type, datetime.strptime(row_text[1], "%A %d %b %Y"))
79
+ )
80
+ collections.append(
81
+ (bin_type, datetime.strptime(row_text[3], "%A %d %b %Y"))
82
+ )
83
+
84
+ ordered_data = sorted(collections, key=lambda x: x[1])
85
+ data = {"bins": []}
86
+ for item in ordered_data:
87
+ dict_data = {
88
+ "type": item[0],
89
+ "collectionDate": item[1].strftime(date_format),
90
+ }
91
+ data["bins"].append(dict_data)
92
+
93
+ return data
@@ -0,0 +1,100 @@
1
+ from bs4 import BeautifulSoup
2
+ from selenium.webdriver.common.by import By
3
+ from selenium.webdriver.support import expected_conditions as EC
4
+ from selenium.webdriver.support.ui import Select
5
+ from selenium.webdriver.support.wait import WebDriverWait
6
+
7
+ from uk_bin_collection.uk_bin_collection.common import *
8
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
9
+
10
+
11
+ # import the wonderful Beautiful Soup and the URL grabber
12
+ class CouncilClass(AbstractGetBinDataClass):
13
+ """
14
+ Concrete classes have to implement all abstract operations of the
15
+ base class. They can also override some operations with a default
16
+ implementation.
17
+ """
18
+
19
+ def parse_data(self, page: str, **kwargs) -> dict:
20
+ driver = None
21
+ try:
22
+ page = "https://selfservice.rushcliffe.gov.uk/renderform.aspx?t=1242&k=86BDCD8DE8D868B9E23D10842A7A4FE0F1023CCA"
23
+
24
+ data = {"bins": []}
25
+
26
+ user_uprn = kwargs.get("uprn")
27
+ user_postcode = kwargs.get("postcode")
28
+ web_driver = kwargs.get("web_driver")
29
+ headless = kwargs.get("headless")
30
+ check_uprn(user_uprn)
31
+ check_postcode(user_postcode)
32
+
33
+ # Create Selenium webdriver
34
+ driver = create_webdriver(web_driver, headless)
35
+ driver.get(page)
36
+
37
+ # Populate postcode field
38
+ inputElement_postcode = driver.find_element(
39
+ By.ID,
40
+ "ctl00_ContentPlaceHolder1_FF3518TB",
41
+ )
42
+ inputElement_postcode.send_keys(user_postcode)
43
+
44
+ # Click search button
45
+ driver.find_element(
46
+ By.ID,
47
+ "ctl00_ContentPlaceHolder1_FF3518BTN",
48
+ ).click()
49
+
50
+ # Wait for the 'Select address' dropdown to appear and select option matching UPRN
51
+ dropdown = WebDriverWait(driver, 10).until(
52
+ EC.presence_of_element_located(
53
+ (By.ID, "ctl00_ContentPlaceHolder1_FF3518DDL")
54
+ )
55
+ )
56
+ # Create a 'Select' for it, then select the matching URPN option
57
+ dropdownSelect = Select(dropdown)
58
+ dropdownSelect.select_by_value("U" + user_uprn)
59
+
60
+ # Wait for the submit button to appear, then click it to get the collection dates
61
+ submit = WebDriverWait(driver, 10).until(
62
+ EC.presence_of_element_located(
63
+ (By.ID, "ctl00_ContentPlaceHolder1_btnSubmit")
64
+ )
65
+ )
66
+ submit.click()
67
+
68
+ soup = BeautifulSoup(driver.page_source, features="html.parser")
69
+
70
+ bins_text = soup.find("div", id="ctl00_ContentPlaceHolder1_pnlConfirmation")
71
+
72
+ if bins_text:
73
+ results = re.findall(
74
+ "Your (.*?) bin will next be collected on (\d\d?\/\d\d?\/\d{4})",
75
+ bins_text.find("div", {"class": "ss_confPanel"}).get_text(),
76
+ )
77
+ if results:
78
+ for result in results:
79
+ collection_date = datetime.strptime(result[1], "%d/%m/%Y")
80
+ dict_data = {
81
+ "type": result[0],
82
+ "collectionDate": collection_date.strftime(date_format),
83
+ }
84
+ data["bins"].append(dict_data)
85
+
86
+ data["bins"].sort(
87
+ key=lambda x: datetime.strptime(
88
+ x.get("collectionDate"), "%d/%m/%Y"
89
+ )
90
+ )
91
+ except Exception as e:
92
+ # Here you can log the exception if needed
93
+ print(f"An error occurred: {e}")
94
+ # Optionally, re-raise the exception if you want it to propagate
95
+ raise
96
+ finally:
97
+ # This block ensures that the driver is closed regardless of an exception
98
+ if driver:
99
+ driver.quit()
100
+ return data
@@ -0,0 +1,81 @@
1
+ from bs4 import BeautifulSoup
2
+ from uk_bin_collection.uk_bin_collection.common import *
3
+ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
4
+ from lxml import etree
5
+
6
+
7
+ # import the wonderful Beautiful Soup and the URL grabber
8
+ class CouncilClass(AbstractGetBinDataClass):
9
+ """
10
+ Concrete classes have to implement all abstract operations of the
11
+ base class. They can also override some operations with a default
12
+ implementation.
13
+ """
14
+
15
+ def parse_data(self, page: str, **kwargs) -> dict:
16
+ # Make a BS4 object
17
+ soup = BeautifulSoup(page.text, features="lxml")
18
+ soup.prettify()
19
+ data = {"bins": []}
20
+ collections = []
21
+
22
+ # Convert the XML to JSON and load the next collection data
23
+ result = soup.find("p").contents[0].text.replace("\\", "")[1:-1]
24
+ json_data = json.loads(result)["NextCollection"]
25
+
26
+ # Get general waste data
27
+ if json_data.get("RefuseCollectionBinDate") is not None:
28
+ bin_type = "Green general waste bin"
29
+ if json_data.get("RefuseBinExceptionMessage") != "":
30
+ bin_type += f" ({json_data.get('RefuseBinExceptionMessage')})".rstrip()
31
+ bin_date = datetime.strptime(
32
+ json_data.get("RefuseCollectionBinDate"), "%Y-%m-%dT%H:%M:%S"
33
+ )
34
+ collections.append((bin_type, bin_date))
35
+
36
+ # Get recycling waste data
37
+ if json_data.get("RecyclingCollectionDate") is not None:
38
+ bin_type = "Blue recycling bin"
39
+ if json_data.get("RecyclingExceptionMessage") != "":
40
+ bin_type += f" ({json_data.get('RecyclingExceptionMessage')})".rstrip()
41
+ bin_date = datetime.strptime(
42
+ json_data.get("RecyclingCollectionDate"), "%Y-%m-%dT%H:%M:%S"
43
+ )
44
+ collections.append((bin_type, bin_date))
45
+
46
+ # Get garden waste data
47
+ if json_data.get("GardenWasteCollectionDate") is not None:
48
+ bin_type = "Brown garden waste bin"
49
+ if json_data.get("GardenWasteExceptionMessage") != "":
50
+ bin_type += (
51
+ f" ({json_data.get('GardenWasteExceptionMessage')})".rstrip()
52
+ )
53
+ bin_date = datetime.strptime(
54
+ json_data.get("GardenWasteCollectionDate"), "%Y-%m-%dT%H:%M:%S"
55
+ )
56
+ collections.append((bin_type, bin_date))
57
+
58
+ # Get food waste data
59
+ if json_data.get("FoodWasteCollectionDate") is not None:
60
+ bin_type = "Black food waste bin"
61
+ if json_data.get("FoodWasteExceptionMessage") != "":
62
+ bin_type += f" ({json_data.get('FoodWasteExceptionMessage')})".rstrip()
63
+ bin_date = datetime.strptime(
64
+ json_data.get("FoodWasteCollectionDate"), "%Y-%m-%dT%H:%M:%S"
65
+ )
66
+ collections.append((bin_type, bin_date))
67
+
68
+ # If there's no collections, raise an error
69
+ if len(collections) < 1:
70
+ raise ValueError("No collections found")
71
+
72
+ # Order the collection by date, then return them
73
+ ordered_data = sorted(collections, key=lambda x: x[1])
74
+ for bin in ordered_data:
75
+ dict_data = {
76
+ "type": bin[0],
77
+ "collectionDate": bin[1].strftime(date_format),
78
+ }
79
+ data["bins"].append(dict_data)
80
+
81
+ return data