warn-scraper 1.2.82__py3-none-any.whl → 1.2.84__py3-none-any.whl

warn/scrapers/md.py

@@ -36,9 +36,13 @@ def scrape(
     # Set the cache
     cache = Cache(cache_dir)
 
+    # In November 2024 Maryland began throwing out many failed connection messages. These two things helped.
+    request_headers = {"User-Agent": "BigLocalNews.org"}
+    request_verify = False
+
     # Get the page
     url = "https://www.dllr.state.md.us/employment/warn.shtml"
-    r = utils.get_url(url)
+    r = utils.get_url(url, headers=request_headers, verify=request_verify)
     r.encoding = "utf-8"
     html = r.text
 
@@ -56,17 +60,41 @@ def scrape(
     html_list = []
     html_list.append(html)  # Save the source HTML for parsing also
 
+    old_pages = [
+        "warn2023.shtml",
+        "warn2022.shtml",
+        "warn2021.shtml",
+        "warn2020.shtml",
+        "warn2019.shtml",
+        "warn2018.shtml",
+        "warn2017.shtml",
+        "warn2016.shtml",
+        "warn2015.shtml",
+        "warn2014.shtml",
+        "warn2013.shtml",
+        "warn2012.shtml",
+        "warn2011.shtml",
+        "warn2010.shtml",
+    ]
+
     for href in href_list:
         # Request the HTML
         url = f"https://www.dllr.state.md.us/employment/{href}"
-        r = utils.get_url(url)
-        r.encoding = "utf-8"
-        html = r.text
-
-        # Save it to the cache
-        cache.write(f"md/{href}.html", html)
-
-        sleep(naptime)  # Try to stop blocked connections by being less aggressive
+        filename = cache_dir / f"md/{href}.html"
+
+        if href not in old_pages:
+            sleep(naptime)  # Try to stop blocked connections by being less aggressive
+            r = utils.get_url(url, headers=request_headers, verify=request_verify)
+            r.encoding = "utf-8"
+            html = r.text
+
+            # Save it to the cache
+            cache.write(filename, html)
+        else:
+            r = utils.fetch_if_not_cached(
+                filename, url, headers=request_headers, verify=request_verify
+            )
+            html = cache.read(filename)
 
         # Add it to the list
         html_list.append(html)

{warn_scraper-1.2.82.dist-info → warn_scraper-1.2.84.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.82
+Version: 1.2.84
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News
@@ -23,7 +23,7 @@ Classifier: Programming Language :: Python :: 3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: click
-Requires-Dist: bs4
+Requires-Dist: beautifulsoup4
 Requires-Dist: html5lib
 Requires-Dist: pdfplumber
 Requires-Dist: requests

{warn_scraper-1.2.82.dist-info → warn_scraper-1.2.84.dist-info}/RECORD

@@ -43,7 +43,7 @@ warn/scrapers/in.py,sha256=dAT40ROhhKiwLcwa_YJ6EyhsYBLe0IX2rOWXmNa6JMs,2026
 warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
 warn/scrapers/ky.py,sha256=XjIojMpaoKbypa7l23IybP02jBijBCJG5UGqfO-EYjg,4365
 warn/scrapers/la.py,sha256=60z-4LZY5xp6aX8r6HGGW3FaOVEGnxlG2Mfgpt4G2WE,12877
-warn/scrapers/md.py,sha256=e-tiiKwr9dNEemtk7SWMv317Nv-qEDf5xPNcMU8AZDQ,3045
+warn/scrapers/md.py,sha256=qRnmYsSFTQWeFUN6RJislnrKJ6ky2tqJUHJrmin9C-s,4011
 warn/scrapers/me.py,sha256=q36F4yJ7hvZsLayA3uBS1romo4X3Qf-sEi2Y7LAQCi8,1172
 warn/scrapers/mi.py,sha256=9clZ9mATEJwdVLzDo_h66rK0aV5Zc7GGQ7AauutS6Wo,3591
 warn/scrapers/mo.py,sha256=wnnwQAiVPwuheMqptMXZpyQdiKNghhKwTO-Bnh9oXoU,3492
@@ -65,9 +65,9 @@ warn/scrapers/va.py,sha256=13lhkQrSkPGHEiWUuf1qiS890PWYE5gV-TgISpoiQnc,1711
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.82.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warn_scraper-1.2.82.dist-info/METADATA,sha256=kxvv_1p1AQwjc40wHLkQddflASZc9I4_GxtcySjdNIo,2025
-warn_scraper-1.2.82.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-warn_scraper-1.2.82.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.82.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
-warn_scraper-1.2.82.dist-info/RECORD,,
+warn_scraper-1.2.84.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warn_scraper-1.2.84.dist-info/METADATA,sha256=Ij4-XcqKM3KulM8QABn24sKO1k23yxuOft-G738JBdY,2036
+warn_scraper-1.2.84.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+warn_scraper-1.2.84.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.84.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
+warn_scraper-1.2.84.dist-info/RECORD,,