warn-scraper 1.2.83__py3-none-any.whl → 1.2.85__py3-none-any.whl

warn/scrapers/md.py

@@ -36,9 +36,13 @@ def scrape(
     # Set the cache
     cache = Cache(cache_dir)
 
+    # In November 2024 Maryland began throwing out many failed connection messages. These two things helped.
+    request_headers = {"User-Agent": "BigLocalNews.org"}
+    request_verify = False
+
     # Get the page
     url = "https://www.dllr.state.md.us/employment/warn.shtml"
-    r = utils.get_url(url)
+    r = utils.get_url(url, headers=request_headers, verify=request_verify)
     r.encoding = "utf-8"
     html = r.text
 
@@ -56,17 +60,41 @@ def scrape(
     html_list = []
     html_list.append(html)  # Save the source HTML for parsing also
 
+    old_pages = [
+        "warn2023.shtml",
+        "warn2022.shtml",
+        "warn2021.shtml",
+        "warn2020.shtml",
+        "warn2019.shtml",
+        "warn2018.shtml",
+        "warn2017.shtml",
+        "warn2016.shtml",
+        "warn2015.shtml",
+        "warn2014.shtml",
+        "warn2013.shtml",
+        "warn2012.shtml",
+        "warn2011.shtml",
+        "warn2010.shtml",
+    ]
+
     for href in href_list:
         # Request the HTML
         url = f"https://www.dllr.state.md.us/employment/{href}"
-        r = utils.get_url(url)
-        r.encoding = "utf-8"
-        html = r.text
-
-        # Save it to the cache
-        cache.write(f"md/{href}.html", html)
-
-        sleep(naptime)  # Try to stop blocked connections by being less aggressive
+        filename = cache_dir / f"md/{href}.html"
+
+        if href not in old_pages:
+            sleep(naptime)  # Try to stop blocked connections by being less aggressive
+            r = utils.get_url(url, headers=request_headers, verify=request_verify)
+            r.encoding = "utf-8"
+            html = r.text
+
+            # Save it to the cache
+            cache.write(filename, html)
+        else:
+            r = utils.fetch_if_not_cached(
+                filename, url, headers=request_headers, verify=request_verify
+            )
+            html = cache.read(filename)
 
         # Add it to the list
         html_list.append(html)

warn/scrapers/tx.py

@@ -31,12 +31,14 @@ def scrape(
 
     Returns: the Path where the file is written
     """
+    ssl_verify = False  # Problems with certificates in November 2024
+
     # Set up the cache
     cache = Cache(cache_dir)
 
     # Get the root URL
     url = "https://www.twc.texas.gov/data-reports/warn-notice"
-    page = utils.get_url(url)
+    page = utils.get_url(url, verify=ssl_verify)
     html = page.text
 
     # Cache it
@@ -68,7 +70,7 @@ def scrape(
         # download the excel file
         year = _get_year(href)
         ext = _get_ext(href)
-        excel_path = cache.download(f"tx/{year}{ext}", data_url)
+        excel_path = cache.download(f"tx/{year}{ext}", data_url, verify=ssl_verify)
 
         # Open it up
         workbook = load_workbook(filename=excel_path)

{warn_scraper-1.2.83.dist-info → warn_scraper-1.2.85.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.83
+Version: 1.2.85
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.83.dist-info → warn_scraper-1.2.85.dist-info}/RECORD

@@ -43,7 +43,7 @@ warn/scrapers/in.py,sha256=dAT40ROhhKiwLcwa_YJ6EyhsYBLe0IX2rOWXmNa6JMs,2026
 warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
 warn/scrapers/ky.py,sha256=XjIojMpaoKbypa7l23IybP02jBijBCJG5UGqfO-EYjg,4365
 warn/scrapers/la.py,sha256=60z-4LZY5xp6aX8r6HGGW3FaOVEGnxlG2Mfgpt4G2WE,12877
-warn/scrapers/md.py,sha256=e-tiiKwr9dNEemtk7SWMv317Nv-qEDf5xPNcMU8AZDQ,3045
+warn/scrapers/md.py,sha256=qRnmYsSFTQWeFUN6RJislnrKJ6ky2tqJUHJrmin9C-s,4011
 warn/scrapers/me.py,sha256=q36F4yJ7hvZsLayA3uBS1romo4X3Qf-sEi2Y7LAQCi8,1172
 warn/scrapers/mi.py,sha256=9clZ9mATEJwdVLzDo_h66rK0aV5Zc7GGQ7AauutS6Wo,3591
 warn/scrapers/mo.py,sha256=wnnwQAiVPwuheMqptMXZpyQdiKNghhKwTO-Bnh9oXoU,3492
@@ -59,15 +59,15 @@ warn/scrapers/ri.py,sha256=EUyLy59eNiYHqiJR8C0YcJrZtp09KyVc45AFD0_Uc0U,4497
 warn/scrapers/sc.py,sha256=p3kscSNSW9C8C5QaSUbCAo6XibgB7G2iH6zaMH7Mnsc,4819
 warn/scrapers/sd.py,sha256=_4R19Ybzsyx1PvcWV3_laJmJ3etrwVGfhNEQm6njwoA,1904
 warn/scrapers/tn.py,sha256=i1H7c09Ea3CDrTXqqRMLBMPT_34QtGA0-x7T8rm_j5Q,2945
-warn/scrapers/tx.py,sha256=7lRIA13CyU1taYdxDA-t6uRn5q13Cr3oR1SNaEe3Dlg,4329
+warn/scrapers/tx.py,sha256=watfR1gyN9w7nluiAOnnIghEmoq3eShNUzYSZ8SkZy4,4438
 warn/scrapers/ut.py,sha256=iUh38YIjbvv5MyyKacsiZNe8KjfdBeDaOf-qMQEF_kc,2245
 warn/scrapers/va.py,sha256=13lhkQrSkPGHEiWUuf1qiS890PWYE5gV-TgISpoiQnc,1711
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.83.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warn_scraper-1.2.83.dist-info/METADATA,sha256=Hm-SxfiTjmUifMl8oXTPkXiCx_QPjpldw9RW8IAlGqY,2036
-warn_scraper-1.2.83.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-warn_scraper-1.2.83.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.83.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
-warn_scraper-1.2.83.dist-info/RECORD,,
+warn_scraper-1.2.85.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warn_scraper-1.2.85.dist-info/METADATA,sha256=lpc4ux36Yj-V8oKjLaWvnofk5MD2ZQGdXG-fIAknW1c,2036
+warn_scraper-1.2.85.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+warn_scraper-1.2.85.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.85.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
+warn_scraper-1.2.85.dist-info/RECORD,,