warn-scraper 1.2.159.dev0__py3-none-any.whl → 1.2.161.dev0__py3-none-any.whl

warn/scrapers/fl.py

@@ -4,8 +4,10 @@ import re
 from os.path import exists
 from pathlib import Path
 
+import niquests
 import pdfplumber
-import requests
+
+# import requests
 import tenacity
 import urllib3
 from bs4 import BeautifulSoup
@@ -51,13 +53,9 @@ def scrape(
     """
     output_csv = data_dir / "fl.csv"
     cache = Cache(cache_dir)  # ~/.warn-scraper/cache
-    # FL site requires realistic User-Agent.
-    headers = {
-        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
-    }
-    # url = "https://floridajobs.org/office-directory/division-of-workforce-services/workforce-programs/reemployment-and-emergency-assistance-coordination-team-react/warn-notices"
+    session = niquests.Session()
     url = "https://floridajobs.org/workforce-resources/worker-adjustment-and-retraining-notification-(warn)"
-    response = requests.get(url, headers=headers, verify=True)
+    response = session.get(url)
     logger.debug(f"Request status is {response.status_code} for {url}")
     html = response.text
     details = pq(html)("div.contact-details")
@@ -83,9 +81,9 @@ def scrape(
     # Loop through years and scrape data
     for year_url in href_lookup.values():
         if "PDF" in year_url:
-            rows_to_add = _scrape_pdf(cache, cache_dir, year_url, headers)
+            rows_to_add = _scrape_pdf(cache, cache_dir, year_url, session)
         else:
-            html_pages = _scrape_html(cache, year_url, headers)
+            html_pages = _scrape_html(cache, year_url, session)
             rows_to_add = _html_to_rows(html_pages)
         # Convert rows to dicts
         rows_as_dicts = [dict(zip(FIELDS, row)) for row in rows_to_add]
@@ -101,9 +99,9 @@ def scrape(
 # note: no max amount of retries (recursive scraping)
 @tenacity.retry(
     wait=tenacity.wait_exponential(),
-    retry=tenacity.retry_if_exception_type(requests.HTTPError),
+    retry=tenacity.retry_if_exception_type(niquests.HTTPError),
 )
-def _scrape_html(cache, url, headers, page=1):
+def _scrape_html(cache, url, session, page=1):
     urllib3.disable_warnings()  # sidestep SSL error
     # extract year from URL
     year = _extract_year(url)
@@ -124,7 +122,7 @@ def _scrape_html(cache, url, headers, page=1):
             raise FileNotFoundError
     except FileNotFoundError:
         # scrape & cache html
-        response = requests.get(url, headers=headers, verify=False)
+        response = session.get(url)
         logger.debug(f"Request status is {response.status_code} for {url}")
         response.raise_for_status()
         page_text = response.text
@@ -145,7 +143,7 @@ def _scrape_html(cache, url, headers, page=1):
                 "href"
             )  # /WarnList/Records?year=XXXX&page=X
             # recursively make list of all the next pages' html
-            pages_html = _scrape_html(cache, url, headers, next_page)
+            pages_html = _scrape_html(cache, url, session, next_page)
             # add the current page to the recursive list
             pages_html.append(page_text)
             return pages_html
@@ -173,9 +171,9 @@ def _html_to_rows(page_text):
 # download and scrape pdf
 @tenacity.retry(
     wait=tenacity.wait_exponential(),
-    retry=tenacity.retry_if_exception_type(requests.HTTPError),
+    retry=tenacity.retry_if_exception_type(niquests.HTTPError),
 )
-def _scrape_pdf(cache, cache_dir, url, headers):
+def _scrape_pdf(cache, cache_dir, url, session):
     # sidestep SSL error
     urllib3.disable_warnings()
     # extract year from URL
@@ -184,7 +182,7 @@ def _scrape_pdf(cache, cache_dir, url, headers):
     download = ""
     # download pdf if not in the cache
     if not exists(pdf_cache_key):
-        response = requests.get(url, headers=headers, verify=False)
+        response = session.get(url)
         logger.debug(f"Request status is {response.status_code} for {url}")
         response.raise_for_status()
         # download & cache pdf

warn/scrapers/md.py

@@ -61,6 +61,8 @@ def scrape(
     html_list.append(html)  # Save the source HTML for parsing also
 
     old_pages = [
+        "warn2025.shtml",
+        "warn2024.shtml",
         "warn2023.shtml",
         "warn2022.shtml",
         "warn2021.shtml",
@@ -83,6 +85,7 @@ def scrape(
         filename = f"md/{href}.html"
 
         if href not in old_pages:
+            logger.debug(f"Trying special handling on {href}")
             sleep(naptime)  # Try to stop blocked connections by being less aggressive
             r = utils.get_url(url, headers=request_headers, verify=request_verify)
             r.encoding = "utf-8"
@@ -116,6 +119,8 @@ def scrape(
         row_list = table.find_all("tr")
 
         # If it's not the first page, slice off the header
+        if i == 0:
+            logger.debug(f"Found possible header row: {row_list[0]}")
         if i > 0:
             row_list = row_list[1:]
 
@@ -123,6 +128,10 @@ def scrape(
         for row in row_list:
             # Get the cells
             cell_list = row.find_all("td")
+            if not cell_list:
+                cell_list = row.find_all(
+                    "th"
+                )  # Alternate approach for first page's header
 
             # Clean them up
             cell_list = [_clean_text(c.text) for c in cell_list]

{warn_scraper-1.2.159.dev0.dist-info → warn_scraper-1.2.161.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.159.dev0
+Version: 1.2.161.dev0
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Author-email: Big Local News <biglocalnews@stanford.edu>
 License-Expression: Apache-2.0

{warn_scraper-1.2.159.dev0.dist-info → warn_scraper-1.2.161.dev0.dist-info}/RECORD

@@ -20,7 +20,7 @@ warn/scrapers/co.py,sha256=83OdikIrWGxt22mlI-_zLSNqJg1NO5C2Xjm3FF6DPYY,18252
 warn/scrapers/ct.py,sha256=PzRnrq91T2a5IzDWMh0GDxWVLXkN-BtHRIYbutjgQhw,2868
 warn/scrapers/dc.py,sha256=p1_c7O2R3O-41DmvcLVUIRhQKUewvZZKkzWkBxytN5M,5165
 warn/scrapers/de.py,sha256=GyM92A-lFwZAfRxgbO-sIWhRfmBEKirzchaPIv-u0o4,1364
-warn/scrapers/fl.py,sha256=jxN7zhlSNgcpBcBB01M2RcYJzE8hUsmKg9zS8AHzWI4,9846
+warn/scrapers/fl.py,sha256=U0XoNQr8S_9IhPutemMpDKSY8IjXf1ZepR-QezGypj4,9410
 warn/scrapers/ga.py,sha256=2kYhuK0Q6tkwp56lTuOwIH7W5zX6KpYJkqsp5ehSdgc,10700
 warn/scrapers/hi.py,sha256=pSplAP15_ZBfQtcywyErmvNcrk7u55TjZj_F0Nqw9L8,5660
 warn/scrapers/ia.py,sha256=0kOh3EOsXtWYGl5kvwimwFZGpOlu5_qFWhM_-n1MLds,2048
@@ -30,7 +30,7 @@ warn/scrapers/in.py,sha256=0hOCdbUyls3xX7Q1z5OB6yCrXJFpsu2CmvfwRh9NTfY,2202
 warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
 warn/scrapers/ky.py,sha256=7kJTNOzxChyXlcyBImmdwwmrczYksU8XNxbhQ2owmJs,9688
 warn/scrapers/la.py,sha256=52i7ICYH0R7omFyu-kmKi_s-8Rhy0vaPjNIKlW_OMC8,6938
-warn/scrapers/md.py,sha256=hwgxXQnhyBWm8qF1dvxIThAX1MkrZbXLwRI9inO5t8g,4060
+warn/scrapers/md.py,sha256=Zjudr5X8-E3fjBXbOqnB830XEZqLznGzQbaJ5HSY0zQ,4425
 warn/scrapers/me.py,sha256=q36F4yJ7hvZsLayA3uBS1romo4X3Qf-sEi2Y7LAQCi8,1172
 warn/scrapers/mi.py,sha256=Ppyawp4nbzSBODuzDKeqnO9_9do5MFwK4Y_f3uc6blE,5846
 warn/scrapers/mo.py,sha256=wnnwQAiVPwuheMqptMXZpyQdiKNghhKwTO-Bnh9oXoU,3492
@@ -54,11 +54,11 @@ warn/scrapers/va.py,sha256=7Nle7qL0VNPiE653XyaP9HQqSfuJFDRr2kEkjOqLvFM,11269
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.159.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
-warn_scraper-1.2.159.dev0.dist-info/METADATA,sha256=_b2Gr_tQPQ9Z5xGk2aBR2FD68gEKPaWIScQCPsPWGxk,1780
-warn_scraper-1.2.159.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-warn_scraper-1.2.159.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.159.dev0.dist-info/scm_file_list.json,sha256=SBROd7oEfcspIMl6v7yfGzQd2rkWxybkiIXopuTQVL0,4120
-warn_scraper-1.2.159.dev0.dist-info/scm_version.json,sha256=lnFVGQ26tYnWMRbtM8vInlCCy60B4aSFEBs6QMOOQFA,161
-warn_scraper-1.2.159.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
-warn_scraper-1.2.159.dev0.dist-info/RECORD,,
+warn_scraper-1.2.161.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
+warn_scraper-1.2.161.dev0.dist-info/METADATA,sha256=-bmCHUv5C4mH2zIWGeKJL_rXeulZVk7G2y4csorP4RA,1780
+warn_scraper-1.2.161.dev0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+warn_scraper-1.2.161.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.161.dev0.dist-info/scm_file_list.json,sha256=SBROd7oEfcspIMl6v7yfGzQd2rkWxybkiIXopuTQVL0,4120
+warn_scraper-1.2.161.dev0.dist-info/scm_version.json,sha256=2zu8ZMmGXe2W6-SIbF-03pXZmDDOzYDnby3i_6KPk80,161
+warn_scraper-1.2.161.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
+warn_scraper-1.2.161.dev0.dist-info/RECORD,,

{warn_scraper-1.2.159.dev0.dist-info → warn_scraper-1.2.161.dev0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (82.0.1)
+Generator: setuptools (83.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

warn_scraper-1.2.161.dev0.dist-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "1.2.160",
+  "distance": 0,
+  "node": "gc0831585f2289f68c9329c4e03d053ec0b80b19e",
+  "dirty": true,
+  "branch": "HEAD",
+  "node_date": "2026-07-05"
+}

warn_scraper-1.2.159.dev0.dist-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "1.2.158",
-  "distance": 0,
-  "node": "g5d6787f288a27ee88a65617b6cd4ffc3b2b475c5",
-  "dirty": true,
-  "branch": "HEAD",
-  "node_date": "2026-06-29"
-}