warn-scraper 1.2.158.dev0__py3-none-any.whl → 1.2.160.dev0__py3-none-any.whl

warn/scrapers/fl.py

@@ -9,6 +9,7 @@ import requests
 import tenacity
 import urllib3
 from bs4 import BeautifulSoup
+from pyquery import PyQuery as pq
 
 from .. import utils
 from ..cache import Cache
@@ -17,7 +18,8 @@ __authors__ = ["zstumgoren", "Dilcia19", "shallotly", "stucka"]
 __tags__ = ["html", "pdf"]
 __source__ = {
     "name": "Florida Department of Economic Opportunity",
-    "url": "https://floridajobs.org/office-directory/division-of-workforce-services/workforce-programs/reemployment-and-emergency-assistance-coordination-team-react/warn-notices",
+    # "url": "https://floridajobs.org/office-directory/division-of-workforce-services/workforce-programs/reemployment-and-emergency-assistance-coordination-team-react/warn-notices",
+    "url": "https://floridajobs.org/workforce-resources/worker-adjustment-and-retraining-notification-(warn)",
 }
 
 logger = logging.getLogger(__name__)
@@ -53,25 +55,24 @@ def scrape(
     headers = {
         "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
     }
-    url = "https://floridajobs.org/office-directory/division-of-workforce-services/workforce-programs/reemployment-and-emergency-assistance-coordination-team-react/warn-notices"
+    # url = "https://floridajobs.org/office-directory/division-of-workforce-services/workforce-programs/reemployment-and-emergency-assistance-coordination-team-react/warn-notices"
+    url = "https://floridajobs.org/workforce-resources/worker-adjustment-and-retraining-notification-(warn)"
     response = requests.get(url, headers=headers, verify=True)
     logger.debug(f"Request status is {response.status_code} for {url}")
-    soup = BeautifulSoup(response.text, "html.parser")
-    pageholder = soup.select("div.mainContentArea")[0]
-    pagesection = pageholder.select("div.sfContentBlock")[0]
+    html = response.text
+    details = pq(html)("div.contact-details")
     href_lookup = {}
-    for atag in pagesection.find_all("a"):
-        href = atag["href"]
+    for link in pq(details)("a.externalIcon"):
+        # logger.debug(pq(link))
+        href = pq(link)("a").attr("href")
         tagyear = href[-4:]
-        if tagyear not in ["orts"]:
+        if tagyear not in ["orts"]:  # exclude "reports"
             href_lookup[tagyear] = href
 
-    # logger.debug(href_lookup)
-    # logger.debug(pagesection)
-
     base_url = "https://reactwarn.floridajobs.org/WarnList/"
 
     # Loop through years and add any missing to the lookup
+    # logger.debug(href_lookup.keys())
     most_recent_year = int(list(href_lookup.keys())[0])
     earliest_year = 2015  # We expected files to be available for at least 2015
     for year in range(earliest_year, most_recent_year):

warn/scrapers/md.py

@@ -61,6 +61,8 @@ def scrape(
     html_list.append(html)  # Save the source HTML for parsing also
 
     old_pages = [
+        "warn2025.shtml",
+        "warn2024.shtml",
         "warn2023.shtml",
         "warn2022.shtml",
         "warn2021.shtml",
@@ -83,6 +85,7 @@ def scrape(
         filename = f"md/{href}.html"
 
         if href not in old_pages:
+            logger.debug(f"Trying special handling on {href}")
             sleep(naptime)  # Try to stop blocked connections by being less aggressive
             r = utils.get_url(url, headers=request_headers, verify=request_verify)
             r.encoding = "utf-8"
@@ -116,6 +119,8 @@ def scrape(
         row_list = table.find_all("tr")
 
         # If it's not the first page, slice off the header
+        if i == 0:
+            logger.debug(f"Found possible header row: {row_list[0]}")
         if i > 0:
             row_list = row_list[1:]
 
@@ -123,6 +128,10 @@ def scrape(
         for row in row_list:
             # Get the cells
             cell_list = row.find_all("td")
+            if not cell_list:
+                cell_list = row.find_all(
+                    "th"
+                )  # Alternate approach for first page's header
 
             # Clean them up
             cell_list = [_clean_text(c.text) for c in cell_list]

warn/scrapers/oh.py

@@ -1,12 +1,13 @@
 import csv
-import json
 import logging
+import re
+from io import StringIO
 from pathlib import Path
 
 import requests
-from bs4 import BeautifulSoup, Tag
 
 from .. import utils
+from ..cache import Cache
 
 __authors__ = ["zstumgoren", "Dilcia19", "chriszs", "stucka"]
 __tags__ = ["html", "pdf"]
@@ -32,6 +33,7 @@ def scrape(
     Returns: the Path where the file is written
     """
     state_code = "oh"
+    cache = Cache()
 
     # Get the latest HTML
     headers = {
@@ -39,29 +41,26 @@ def scrape(
     }
 
     # latesturl = "https://jfs.ohio.gov/wps/portal/gov/jfs/job-services-and-unemployment/job-services/job-programs-and-services/submit-a-warn-notice/current-public-notices-of-layoffs-and-closures-sa/current-public-notices-of-layoffs-and-closures"
-    latesturl = "https://jfs.ohio.gov/job-services-and-unemployment/job-services/job-programs-and-services/submit-a-warn-notice/current-public-notices-of-layoffs-and-closures-sa"
-
+    # latesturl = "https://jfs.ohio.gov/job-services-and-unemployment/job-services/job-programs-and-services/submit-a-warn-notice/current-public-notices-of-layoffs-and-closures-sa"
+    latesturl = "https://jfs.ohio.gov/job-workforce-services/job-programs-and-services/submit-a-warn-notice/current-public-notices-of-layoffs-and-closures"
     logger.debug(f"Attempting to fetch current data from {latesturl}")
     r = requests.get(latesturl, headers=headers)
-    soup = BeautifulSoup(r.content, features="lxml")
-    logger.debug("Attempting to get JSON data from Ohio file")
-    data_div = soup.find("div", {"id": "js-placeholder-json-data"})
-    if isinstance(data_div, Tag):
-        data = json.loads(data_div.decode_contents().strip())["data"]
-    else:
-        logger.debug("!!!!!!!!!!!!!!!!!!!! Could not find JSON data div")
-        logger.debug(soup)
-        raise ValueError("Could not find JSON data div")
-    rawheaders = data[1]
-    logger.debug(f"Found headers: {rawheaders}")
-    masterlist = []
-    for row in data[2:]:
-        if len(row) == len(rawheaders):
-            line = {}
-            for i, item in enumerate(rawheaders):
-                if item != "":
-                    line[item] = row[i]
-            masterlist.append(line)
+    cache.write("oh/index.html", r.text)
+    logger.debug("Attempting to get CSV link from Ohio file")
+    html = r.text
+    csv_url = re.findall(r"(\\\"csvUrl\\\":\\\")(.*?)(\\\")", html)[0][1]
+    logger.debug(f"CSV link found at {csv_url}")
+    r = requests.get(csv_url, headers=headers)
+    cache.write("oh/rawdata.csv", r.text)
+
+    # Ohio CSV as of June 2026 was coming in with extra prefacing headers that ... make it not a CSV.
+    # So knock off any lines without useful data.
+    clean = []
+    for myline in r.text.splitlines():
+        if len(myline) > 20:
+            clean.append(myline)
+
+    masterlist = list(csv.DictReader(StringIO("\n".join(clean))))
 
     logger.debug("Get historical data and meld it into current format")
     # Get the historical data, and meld it into the same format
@@ -82,7 +81,7 @@ def scrape(
     )
     reader = list(csv.DictReader(r.text.splitlines()))
     for row in reader:
-        line = {}
+        line: dict = {}
         for item in lookup:
             if not lookup[item]:
                 line[item] = None

{warn_scraper-1.2.158.dev0.dist-info → warn_scraper-1.2.160.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.158.dev0
+Version: 1.2.160.dev0
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Author-email: Big Local News <biglocalnews@stanford.edu>
 License-Expression: Apache-2.0

{warn_scraper-1.2.158.dev0.dist-info → warn_scraper-1.2.160.dev0.dist-info}/RECORD

@@ -20,7 +20,7 @@ warn/scrapers/co.py,sha256=83OdikIrWGxt22mlI-_zLSNqJg1NO5C2Xjm3FF6DPYY,18252
 warn/scrapers/ct.py,sha256=PzRnrq91T2a5IzDWMh0GDxWVLXkN-BtHRIYbutjgQhw,2868
 warn/scrapers/dc.py,sha256=p1_c7O2R3O-41DmvcLVUIRhQKUewvZZKkzWkBxytN5M,5165
 warn/scrapers/de.py,sha256=GyM92A-lFwZAfRxgbO-sIWhRfmBEKirzchaPIv-u0o4,1364
-warn/scrapers/fl.py,sha256=pBkDo3mcKIjXSkWf469QCGLuyn7gH1YfNeX0AwVsuL8,9642
+warn/scrapers/fl.py,sha256=jxN7zhlSNgcpBcBB01M2RcYJzE8hUsmKg9zS8AHzWI4,9846
 warn/scrapers/ga.py,sha256=2kYhuK0Q6tkwp56lTuOwIH7W5zX6KpYJkqsp5ehSdgc,10700
 warn/scrapers/hi.py,sha256=pSplAP15_ZBfQtcywyErmvNcrk7u55TjZj_F0Nqw9L8,5660
 warn/scrapers/ia.py,sha256=0kOh3EOsXtWYGl5kvwimwFZGpOlu5_qFWhM_-n1MLds,2048
@@ -30,7 +30,7 @@ warn/scrapers/in.py,sha256=0hOCdbUyls3xX7Q1z5OB6yCrXJFpsu2CmvfwRh9NTfY,2202
 warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
 warn/scrapers/ky.py,sha256=7kJTNOzxChyXlcyBImmdwwmrczYksU8XNxbhQ2owmJs,9688
 warn/scrapers/la.py,sha256=52i7ICYH0R7omFyu-kmKi_s-8Rhy0vaPjNIKlW_OMC8,6938
-warn/scrapers/md.py,sha256=hwgxXQnhyBWm8qF1dvxIThAX1MkrZbXLwRI9inO5t8g,4060
+warn/scrapers/md.py,sha256=Zjudr5X8-E3fjBXbOqnB830XEZqLznGzQbaJ5HSY0zQ,4425
 warn/scrapers/me.py,sha256=q36F4yJ7hvZsLayA3uBS1romo4X3Qf-sEi2Y7LAQCi8,1172
 warn/scrapers/mi.py,sha256=Ppyawp4nbzSBODuzDKeqnO9_9do5MFwK4Y_f3uc6blE,5846
 warn/scrapers/mo.py,sha256=wnnwQAiVPwuheMqptMXZpyQdiKNghhKwTO-Bnh9oXoU,3492
@@ -40,7 +40,7 @@ warn/scrapers/ne.py,sha256=JawuGJ3tCKvMd-N-p03gnltB4rol4QUJshMk2oyMPO4,4143
 warn/scrapers/nj.py,sha256=nwbMbeQuUJbYRVoyUyKZBmNqvqsXu3Habt-10r8DvZE,2230
 warn/scrapers/nm.py,sha256=55kDh65uOc-uMO8wuxPHhHN4Aw8Di2Fx66JBds8f2Io,3694
 warn/scrapers/ny.py,sha256=_nYoawIde4yYtk8-tMG3hekLlSlrKKqULmkBWOUoQJM,2586
-warn/scrapers/oh.py,sha256=-_MyuaUGyKgd-XiXMVnf6AHkqOBgAFg0NRRpXGLz-3A,3431
+warn/scrapers/oh.py,sha256=kwHm1feWQwZRdWKnZbDm1c2wdYqNUHJ1myrK6uteyfU,3502
 warn/scrapers/ok.py,sha256=ZZciyR1jPS4SzS2JSQwhJsDXP_VxA9UkEQvLpxzWzp4,7676
 warn/scrapers/or.py,sha256=0PjyrW3CHdxtHhqEo3Ob-9B6YckACoBD3K0c4FPQUcg,5208
 warn/scrapers/pa.py,sha256=j6cYZjKJUht2s829VPAbI2IRJazkqOwbwcVMAcItuIc,5410
@@ -54,11 +54,11 @@ warn/scrapers/va.py,sha256=7Nle7qL0VNPiE653XyaP9HQqSfuJFDRr2kEkjOqLvFM,11269
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.158.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
-warn_scraper-1.2.158.dev0.dist-info/METADATA,sha256=eEAoxrMSjuAIXEEF7mpOOan9f74w4CacJtNiWOxPM6E,1780
-warn_scraper-1.2.158.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-warn_scraper-1.2.158.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.158.dev0.dist-info/scm_file_list.json,sha256=SBROd7oEfcspIMl6v7yfGzQd2rkWxybkiIXopuTQVL0,4120
-warn_scraper-1.2.158.dev0.dist-info/scm_version.json,sha256=XFb9EYQ2iJW5cLXZGITnEVHhw7wZNgPoYMAsYnRCNbU,161
-warn_scraper-1.2.158.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
-warn_scraper-1.2.158.dev0.dist-info/RECORD,,
+warn_scraper-1.2.160.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
+warn_scraper-1.2.160.dev0.dist-info/METADATA,sha256=eHYbY-2hpPWQAbxPEdBufp79t3vSb1e0gE0WPRKAtMY,1780
+warn_scraper-1.2.160.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+warn_scraper-1.2.160.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.160.dev0.dist-info/scm_file_list.json,sha256=SBROd7oEfcspIMl6v7yfGzQd2rkWxybkiIXopuTQVL0,4120
+warn_scraper-1.2.160.dev0.dist-info/scm_version.json,sha256=BlfOZmrMfTxErAqmqWjuNwb0FtN0WDp9xyU4qysIay8,161
+warn_scraper-1.2.160.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
+warn_scraper-1.2.160.dev0.dist-info/RECORD,,

warn_scraper-1.2.160.dev0.dist-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "1.2.159",
+  "distance": 0,
+  "node": "gaaf920ffa9b99245ac689d6d7ef4c1e29fd4c7dc",
+  "dirty": true,
+  "branch": "HEAD",
+  "node_date": "2026-07-03"
+}

warn_scraper-1.2.158.dev0.dist-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "1.2.157",
-  "distance": 0,
-  "node": "g2dc89394a4e2daac88ce2f78548663e8b41d2096",
-  "dirty": true,
-  "branch": "HEAD",
-  "node_date": "2026-06-24"
-}