warn-scraper 1.2.156.dev0__py3-none-any.whl → 1.2.158.dev0__py3-none-any.whl

warn/scrapers/al.py

@@ -1,14 +1,14 @@
+import csv
 import logging
-import re
+from io import StringIO
 from pathlib import Path
 
-from bs4 import BeautifulSoup
-
 from .. import utils
+from ..cache import Cache
 
-__authors__ = ["zstumgoren", "Dilcia19"]
+__authors__ = ["zstumgoren", "Dilcia19", "stucka"]
 __tags__ = [
-    "html",
+    "csv",
 ]
 __source__ = {
     "name": "Alabama Department of Commerce",
@@ -31,45 +31,40 @@ def scrape(
 
     Returns: the Path where the file is written
     """
+    cache = Cache()
     output_csv = data_dir / "al.csv"
+
     # page = utils.get_url("https://www.madeinalabama.com/warn-list/")
     # URL change in June 2026, maybe led to a HTTP 415 error
-    page = utils.get_url("https://workforce.alabama.gov/warn-list/")
-
-    # can't see 2020 listings when I open web page, but they are on the summary in the google search
-    soup = BeautifulSoup(page.text, "html.parser")
-    table = soup.find_all("table")  # output is list-type
-    table_rows = table[0].find_all("tr")
-    logger.debug(f"{len(table_rows):,} total table rows (including header) found")
-    # Handle the header
-    raw_header = table_rows.pop(0)
-    header_row = _extract_fields_from_row(raw_header, "th")
-    output_rows = [header_row]
-    # Process remaining rows
-    discarded_rows = []
-    for table_row in table_rows:
-        # Discard bogus data lines (see last lines of source data)
-        # based on check of first field ("Closing or Layoff")
-        data = _extract_fields_from_row(table_row, "td")
-        layoff_type = data[0]
-        if re.match(r"(clos|lay)", layoff_type, re.I):
-            output_rows.append(data)
-        else:
-            discarded_rows.append(data)
-    if discarded_rows:
-        logger.warn(f"Warning: Discarded {len(discarded_rows)} dirty data row(s)")
-    utils.write_rows_to_csv(output_csv, output_rows)
-    return output_csv
+    # page = utils.get_url("https://workforce.alabama.gov/warn-list/")
+    # Later in June 2026, they're detecting the automation and blocking us.
+    # But say they won't for the CSV download which, OK.
+    # No headers on the CSV, which they'll probably realize ... sometime.
+
+    targeturl = "https://workforce.alabama.gov/documents/warn-list/"
+    page = utils.get_url(targeturl).text
+    cache.write("al/rawcsv.csv", page)
 
+    headers = [
+        "_id1",
+        "action_type",
+        "date_notice",
+        "date_action",
+        "company",
+        "location",
+        "affected",
+        "_id2",
+    ]
 
-def _extract_fields_from_row(row, element):
-    """Pluck data from the provided row and element."""
-    row_data = []
-    fields = row.find_all(element)
-    for raw_field in fields:
-        field = raw_field.text.strip()
-        row_data.append(field)
-    return row_data
+    fileholder = StringIO(page)
+
+    reader = list(csv.DictReader(fileholder, fieldnames=headers))
+
+    utils.write_disparate_dict_rows_to_csv(
+        output_csv, reader, mode="w", prefixes=["_id"]
+    )
+
+    return output_csv
 
 
 if __name__ == "__main__":

{warn_scraper-1.2.156.dev0.dist-info → warn_scraper-1.2.158.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.156.dev0
+Version: 1.2.158.dev0
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Author-email: Big Local News <biglocalnews@stanford.edu>
 License-Expression: Apache-2.0

{warn_scraper-1.2.156.dev0.dist-info → warn_scraper-1.2.158.dev0.dist-info}/RECORD

@@ -13,7 +13,7 @@ warn/platforms/job_center/urls.py,sha256=IWhpuzN_xcNdHh23GbZPGvuHCsMcmb03qx3pRn1
 warn/platforms/job_center/utils.py,sha256=HdUKgKirmpPP7e4Cu_ZyB3zPVS_p-_ylo-lXFhxK2QM,5696
 warn/scrapers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warn/scrapers/ak.py,sha256=h7BYMTV0whwWAPhbzVDVKMMoVCFphKly70aiTHabPq4,1847
-warn/scrapers/al.py,sha256=XSDEGC7F6_3GZ2m_uSiIG-1v8jMNH9pw_wNUCZyaMK0,2460
+warn/scrapers/al.py,sha256=3_OtgUp4K6OL6H63SdtLQWE998r8PQpTNOv6PuiJEEc,1888
 warn/scrapers/az.py,sha256=elGbue01Gjf_DQ66Wy9qqGIOJsiY-KIKJOVeft8pCXg,1447
 warn/scrapers/ca.py,sha256=VQOfjHXPCc-jYwh-EPGVVfnzvXB7pdmCt2uJ6QnMPRM,8600
 warn/scrapers/co.py,sha256=83OdikIrWGxt22mlI-_zLSNqJg1NO5C2Xjm3FF6DPYY,18252
@@ -54,9 +54,11 @@ warn/scrapers/va.py,sha256=7Nle7qL0VNPiE653XyaP9HQqSfuJFDRr2kEkjOqLvFM,11269
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.156.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
-warn_scraper-1.2.156.dev0.dist-info/METADATA,sha256=3ZYa-9I2ULmcbnva6Uyjqw5XNgxv4Lc5LetTtWT0w_s,1780
-warn_scraper-1.2.156.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-warn_scraper-1.2.156.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.156.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
-warn_scraper-1.2.156.dev0.dist-info/RECORD,,
+warn_scraper-1.2.158.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
+warn_scraper-1.2.158.dev0.dist-info/METADATA,sha256=eEAoxrMSjuAIXEEF7mpOOan9f74w4CacJtNiWOxPM6E,1780
+warn_scraper-1.2.158.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+warn_scraper-1.2.158.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.158.dev0.dist-info/scm_file_list.json,sha256=SBROd7oEfcspIMl6v7yfGzQd2rkWxybkiIXopuTQVL0,4120
+warn_scraper-1.2.158.dev0.dist-info/scm_version.json,sha256=XFb9EYQ2iJW5cLXZGITnEVHhw7wZNgPoYMAsYnRCNbU,161
+warn_scraper-1.2.158.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
+warn_scraper-1.2.158.dev0.dist-info/RECORD,,

warn_scraper-1.2.158.dev0.dist-info/scm_file_list.json

@@ -0,0 +1,137 @@
+{
+  "files": [
+    ".pre-commit-config.yaml",
+    "README.md",
+    "Makefile",
+    "setup.cfg",
+    "LICENSE",
+    "setup.py",
+    "pyproject.toml",
+    "Pipfile.lock",
+    "MANIFEST.in",
+    "Pipfile",
+    ".gitignore",
+    ".devcontainer/devcontainer.json",
+    "docs/reference.rst",
+    "docs/releasing.md",
+    "docs/Makefile",
+    "docs/sources.md",
+    "docs/contributing.rst",
+    "docs/index.rst",
+    "docs/requirements.txt",
+    "docs/make.bat",
+    "docs/usage.md",
+    "docs/conf.py",
+    "docs/scrapers/az.md",
+    "docs/scrapers/de.md",
+    "docs/scrapers/or.md",
+    "docs/scrapers/tx.md",
+    "docs/scrapers/sc.md",
+    "docs/scrapers/vt.md",
+    "docs/scrapers/job_center.md",
+    "docs/scrapers/dc.md",
+    "docs/scrapers/ut.md",
+    "docs/scrapers/mo.md",
+    "docs/scrapers/al.md",
+    "docs/scrapers/me.md",
+    "docs/scrapers/ks.md",
+    "docs/scrapers/in.md",
+    "docs/scrapers/ok.md",
+    "docs/scrapers/co.md",
+    "docs/scrapers/ia.md",
+    "docs/scrapers/va.md",
+    "docs/scrapers/md.md",
+    "docs/scrapers/ny.md",
+    "docs/scrapers/wi.md",
+    "docs/scrapers/ca.md",
+    "docs/_static/releasing-changelog-entered.png",
+    "docs/_static/releasing-publish-button.png",
+    "docs/_static/releasing-actions-start.png",
+    "docs/_static/gao-03-1003.pdf",
+    "docs/_static/releasing-tag-button.png",
+    "docs/_static/releasing-name-tag.png",
+    "docs/_static/releasing-draft-button.png",
+    "docs/_static/releasing-releases-button.png",
+    "docs/_static/R42693.pdf",
+    "docs/_static/releasing-actions-finished.png",
+    "docs/_static/releasing-pypi.png",
+    "docs/_static/releasing-name-release.png",
+    "docs/_static/releasing-release-published.png",
+    "docs/_static/releasing-changelog-button.png",
+    "docs/_templates/sources.md.tmpl",
+    "warn/__init__.py",
+    "warn/utils.py",
+    "warn/runner.py",
+    "warn/cache.py",
+    "warn/cli.py",
+    "warn/scrapers/az.py",
+    "warn/scrapers/la.py",
+    "warn/scrapers/tx.py",
+    "warn/scrapers/wa.py",
+    "warn/scrapers/wi.py",
+    "warn/scrapers/ga.py",
+    "warn/scrapers/co.py",
+    "warn/scrapers/de.py",
+    "warn/scrapers/__init__.py",
+    "warn/scrapers/ia.py",
+    "warn/scrapers/al.py",
+    "warn/scrapers/fl.py",
+    "warn/scrapers/nj.py",
+    "warn/scrapers/mi.py",
+    "warn/scrapers/vt.py",
+    "warn/scrapers/mt.py",
+    "warn/scrapers/md.py",
+    "warn/scrapers/sc.py",
+    "warn/scrapers/ut.py",
+    "warn/scrapers/il.py",
+    "warn/scrapers/ky.py",
+    "warn/scrapers/or.py",
+    "warn/scrapers/in.py",
+    "warn/scrapers/ca.py",
+    "warn/scrapers/hi.py",
+    "warn/scrapers/ak.py",
+    "warn/scrapers/ri.py",
+    "warn/scrapers/ny.py",
+    "warn/scrapers/pa.py",
+    "warn/scrapers/dc.py",
+    "warn/scrapers/ct.py",
+    "warn/scrapers/mo.py",
+    "warn/scrapers/ok.py",
+    "warn/scrapers/tn.py",
+    "warn/scrapers/ks.py",
+    "warn/scrapers/oh.py",
+    "warn/scrapers/id.py",
+    "warn/scrapers/nm.py",
+    "warn/scrapers/me.py",
+    "warn/scrapers/va.py",
+    "warn/scrapers/ne.py",
+    "warn/scrapers/sd.py",
+    "warn/scrapers/ms.py",
+    "warn/platforms/__init__.py",
+    "warn/platforms/job_center/__init__.py",
+    "warn/platforms/job_center/utils.py",
+    "warn/platforms/job_center/cache.py",
+    "warn/platforms/job_center/site.py",
+    "warn/platforms/job_center/urls.py",
+    "warn/pdfrodent/__init__.py",
+    "warn/pdfrodent/pdfrodent.py",
+    "tests/__init__.py",
+    "tests/test_job_center.py",
+    "tests/test_job_center_cache.py",
+    "tests/test_openpyxl.py",
+    "tests/conftest.py",
+    "tests/test_cache.py",
+    "tests/test_delete.py",
+    "tests/fixtures/2021_page_2.html",
+    "tests/fixtures/2021_page_1.html",
+    "tests/cassettes/test_scrape_integration.yaml",
+    "tests/cassettes/test_cached_search_results.yaml",
+    "tests/cassettes/test_no_results.yaml",
+    "tests/cassettes/test_paged_results.yaml",
+    "tests/cassettes/test_missing_detail_page_values.yaml",
+    "tests/cassettes/test_cached_detail_pages.yaml",
+    ".github/dependabot-disabled.yml",
+    ".github/workflows/continuous-deployment.yml",
+    ".github/workflows/continuous-deployment.yml.broken-tests"
+  ]
+}

warn_scraper-1.2.158.dev0.dist-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "1.2.157",
+  "distance": 0,
+  "node": "g2dc89394a4e2daac88ce2f78548663e8b41d2096",
+  "dirty": true,
+  "branch": "HEAD",
+  "node_date": "2026-06-24"
+}