warn-scraper 1.2.169.dev0__py3-none-any.whl → 1.2.170.dev0__py3-none-any.whl

warn/scrapers/ma.py

@@ -0,0 +1,271 @@
+import csv
+import logging
+import os
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import unquote, urljoin
+
+from bs4 import BeautifulSoup
+from openpyxl import load_workbook
+
+from .. import utils
+from ..cache import Cache
+
+__authors__ = ["riordan"]
+__tags__ = ["excel", "csv", "historical"]
+__source__ = {
+    "name": "Massachusetts Executive Office of Labor and Workforce Development",
+    "url": "https://www.mass.gov/info-details/worker-adjustment-and-retraining-notification-act-warn-layoff-and-closure-updates",
+}
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://www.mass.gov"
+
+# The unified header for the combined export. It mirrors the columns the state
+# currently publishes in its flat fiscal-year workbooks and its weekly CSV feed.
+CANONICAL_HEADER = [
+    "RECEIVED",
+    "EMPLOYER",
+    "CITY/TOWN",
+    "REGION",
+    "DATE(S) OF LAYOFFS",
+    "# EMPLOYEES IMPACTED",
+]
+
+# The older fiscal-year workbooks (FY2022, FY2023) split notices into one
+# worksheet per region and lay the columns out in this fixed order. The region
+# for those rows comes from the worksheet name rather than a column.
+REGION_SHEET_ORDER = [
+    "RECEIVED",
+    "EMPLOYER",
+    "CITY/TOWN",
+    "DATE(S) OF LAYOFFS",
+    "# EMPLOYEES IMPACTED",
+]
+
+
+def scrape(
+    data_dir: Path = utils.WARN_DATA_DIR,
+    cache_dir: Path = utils.WARN_CACHE_DIR,
+) -> Path:
+    """
+    Scrape data from Massachusetts.
+
+    Keyword arguments:
+    data_dir -- the Path were the result will be saved (default WARN_DATA_DIR)
+    cache_dir -- the Path where results can be cached (default WARN_CACHE_DIR)
+
+    Returns: the Path where the file is written
+    """
+    # Fire up the cache
+    cache = Cache(cache_dir)
+    state_code = "ma"
+
+    # Download the index page. Mass.gov sits behind Akamai bot protection that
+    # rejects ordinary requests, so we route everything through Zyte, as the
+    # Louisiana and Texas scrapers do.
+    _, html = utils.get_with_zyte(__source__["url"])
+    cache.write(f"{state_code}/index.html", html)
+
+    # Find the links to the fiscal-year workbooks and the weekly CSV feed.
+    soup = BeautifulSoup(html, "html.parser")
+    excel_urls, csv_urls = _find_source_links(soup)
+    logger.debug(
+        f"Found {len(excel_urls)} workbook link(s) and {len(csv_urls)} CSV link(s)"
+    )
+
+    master_list: list = []
+
+    # The fiscal-year workbooks hold the historical record.
+    for url in excel_urls:
+        name = url.split("/doc/")[1].split("/")[0]
+        raw, _ = utils.get_with_zyte(url)
+        excel_path = cache.write_binary(f"{state_code}/{name}.xlsx", raw)
+        master_list.extend(_parse_workbook(excel_path))
+
+    # The weekly CSV feed carries the current fiscal year's running notices,
+    # which are not yet folded into a workbook.
+    for url in csv_urls:
+        name = os.path.basename(unquote(url))
+        raw, text = utils.get_with_zyte(url)
+        cache.write_binary(f"{state_code}/{name}", raw)
+        rows = list(csv.reader(text.splitlines()))
+        master_list.extend(_parse_flat_rows(rows))
+
+    # Drop exact duplicate rows while preserving order. Duplicates can appear at
+    # fiscal-year boundaries or across re-runs; distinct notices are untouched.
+    deduped = _dedupe(master_list)
+    logger.debug(
+        f"Parsed {len(master_list):,} rows, {len(deduped):,} after removing duplicates"
+    )
+
+    # Write it out
+    data_path = data_dir / f"{state_code}.csv"
+    utils.write_dict_rows_to_csv(
+        data_path, CANONICAL_HEADER, deduped, extrasaction="raise"
+    )
+
+    return data_path
+
+
+def _find_source_links(soup: BeautifulSoup) -> tuple:
+    """Pull the workbook and CSV download links off the index page.
+
+    Args:
+        soup (BeautifulSoup): the parsed index page
+
+    Returns: a tuple of (sorted workbook urls, sorted csv urls)
+    """
+    excel_urls = set()
+    csv_urls = set()
+    for link in soup.find_all("a", href=True):
+        href = link["href"]
+        lower = href.lower()
+        if "http" not in lower:
+            href = urljoin(BASE_URL, href)
+        # Fiscal-year workbooks live at /doc/fyNN-warn-report.../download
+        if "/doc/" in lower and "warn-report" in lower:
+            excel_urls.add(href)
+        # The weekly feed is a WARN Report CSV under /files/
+        elif lower.endswith(".csv") and "warn" in lower:
+            csv_urls.add(href)
+    return sorted(excel_urls), sorted(csv_urls)
+
+
+def _parse_workbook(excel_path: Path) -> list:
+    """Parse a fiscal-year workbook into a list of canonical row dicts.
+
+    Args:
+        excel_path (Path): the path to a cached .xlsx file
+
+    Returns: a list of dicts keyed by CANONICAL_HEADER
+    """
+    workbook = load_workbook(filename=excel_path, read_only=True, data_only=True)
+    rows: list = []
+    for sheet in workbook.worksheets:
+        sheet_rows = [list(r) for r in sheet.iter_rows(values_only=True)]
+        if _has_region_column(sheet_rows):
+            rows.extend(_parse_flat_rows(sheet_rows))
+        else:
+            rows.extend(_parse_region_sheet(sheet_rows, sheet.title.strip()))
+    workbook.close()
+    return rows
+
+
+def _has_region_column(rows: list) -> bool:
+    """Return True if the sheet is a flat table with its own REGION column."""
+    header_index = _find_header_row(rows)
+    if header_index is None:
+        return False
+    return any(_norm(cell) == "REGION" for cell in rows[header_index])
+
+
+def _find_header_row(rows: list):
+    """Find the index of the header row, or None if there isn't one."""
+    for i, row in enumerate(rows):
+        labels = {_norm(cell) for cell in row}
+        if "EMPLOYER" in labels or "COMPANY NAME" in labels:
+            return i
+    return None
+
+
+def _parse_flat_rows(rows: list) -> list:
+    """Parse a flat table (FY2024+ workbooks and the weekly CSV) into row dicts.
+
+    The header row names its own columns, including REGION, so we map each
+    canonical field to the column where its header appears.
+    """
+    header_index = _find_header_row(rows)
+    if header_index is None:
+        return []
+
+    # Map each canonical column name to its position in the header row.
+    positions = {}
+    for col, cell in enumerate(rows[header_index]):
+        label = _norm(cell)
+        if label in CANONICAL_HEADER and label not in positions:
+            positions[label] = col
+
+    parsed = []
+    for row in rows[header_index + 1 :]:
+        record = {
+            field: _clean(row[pos]) if pos < len(row) else ""
+            for field, pos in positions.items()
+        }
+        record = {field: record.get(field, "") for field in CANONICAL_HEADER}
+        if _keep_row(record):
+            parsed.append(record)
+    return parsed
+
+
+def _parse_region_sheet(rows: list, region: str) -> list:
+    """Parse an older per-region worksheet (FY2022, FY2023) into row dicts.
+
+    These sheets have no REGION column and lay their columns out positionally,
+    so the region comes from the worksheet name.
+    """
+    header_index = _find_header_row(rows)
+    if header_index is None:
+        return []
+
+    parsed = []
+    for row in rows[header_index + 1 :]:
+        record = {field: "" for field in CANONICAL_HEADER}
+        for pos, field in enumerate(REGION_SHEET_ORDER):
+            if pos < len(row):
+                record[field] = _clean(row[pos])
+        record["REGION"] = region
+        if _keep_row(record):
+            parsed.append(record)
+    return parsed
+
+
+def _keep_row(record: dict) -> bool:
+    """Decide whether a parsed row is a real notice worth keeping."""
+    employer = record.get("EMPLOYER", "")
+    if not employer:
+        return False
+    # Skip stray header echoes and any total/count footer rows.
+    if _norm(employer) in ("EMPLOYER", "COMPANY NAME"):
+        return False
+    if _norm(employer).startswith("TOTAL"):
+        return False
+    return True
+
+
+def _dedupe(rows: list) -> list:
+    """Drop exact duplicate rows while preserving their first-seen order."""
+    seen = set()
+    deduped = []
+    for row in rows:
+        key = tuple(row[field] for field in CANONICAL_HEADER)
+        if key not in seen:
+            seen.add(key)
+            deduped.append(row)
+    return deduped
+
+
+def _clean(value) -> str:
+    """Normalize a cell value to a clean string for output.
+
+    Dates arrive from openpyxl as datetime objects; render them the way the
+    state writes them elsewhere (M/D/YYYY). Everything else is stringified and
+    stripped.
+    """
+    if value is None:
+        return ""
+    if isinstance(value, datetime):
+        return f"{value.month}/{value.day}/{value.year}"
+    return str(value).strip()
+
+
+def _norm(value) -> str:
+    """Uppercase, whitespace-collapsed form of a cell, for label matching."""
+    if value is None:
+        return ""
+    return " ".join(str(value).split()).upper()
+
+
+if __name__ == "__main__":
+    scrape()

warn/scrapers/nd.py

@@ -48,7 +48,7 @@ def scrape(
         for page_index, page in enumerate(pdf.pages):
             rows = page.extract_table() or []
 
-            for row in rows:
+            for rowindex, row in enumerate(rows):
                 # Standardize each cell
                 output_row = [_clean_text(cell) for cell in row]
 
@@ -65,6 +65,9 @@ def scrape(
                         continue
                     header_written = True
 
+                if rowindex > 0:  # If not a header row
+                    if output_row[1].endswith(", ND"):
+                        output_row[1] = output_row[1][:-4]
                 output_rows.append(output_row)
 
     # Write out to CSV

{warn_scraper-1.2.169.dev0.dist-info → warn_scraper-1.2.170.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.169.dev0
+Version: 1.2.170.dev0
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Author-email: Big Local News <biglocalnews@stanford.edu>
 License-Expression: Apache-2.0

{warn_scraper-1.2.169.dev0.dist-info → warn_scraper-1.2.170.dev0.dist-info}/RECORD

@@ -30,13 +30,14 @@ warn/scrapers/in.py,sha256=0hOCdbUyls3xX7Q1z5OB6yCrXJFpsu2CmvfwRh9NTfY,2202
 warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
 warn/scrapers/ky.py,sha256=7kJTNOzxChyXlcyBImmdwwmrczYksU8XNxbhQ2owmJs,9688
 warn/scrapers/la.py,sha256=52i7ICYH0R7omFyu-kmKi_s-8Rhy0vaPjNIKlW_OMC8,6938
+warn/scrapers/ma.py,sha256=oJa3GMBm2YsG2nFqGbb2ZKKhurfN59ETeTcLABcVF00,8809
 warn/scrapers/md.py,sha256=Zjudr5X8-E3fjBXbOqnB830XEZqLznGzQbaJ5HSY0zQ,4425
 warn/scrapers/me.py,sha256=q36F4yJ7hvZsLayA3uBS1romo4X3Qf-sEi2Y7LAQCi8,1172
 warn/scrapers/mi.py,sha256=Ppyawp4nbzSBODuzDKeqnO9_9do5MFwK4Y_f3uc6blE,5846
 warn/scrapers/mo.py,sha256=wnnwQAiVPwuheMqptMXZpyQdiKNghhKwTO-Bnh9oXoU,3492
 warn/scrapers/ms.py,sha256=BZZoMw3TNtwzBeBsqKLsPznBcDjPaO49I5-yBU0e9AI,5502
 warn/scrapers/mt.py,sha256=t2MP4OCcuCEnrnvNgOu289P0eekZq4XaCK65qzgZX88,2457
-warn/scrapers/nd.py,sha256=x-mLrysn-Ns3tXg6MmjUYZ-4T0x6YUix_fuIldO5Dk0,2657
+warn/scrapers/nd.py,sha256=3hXgOpu59QElYf44mFWYlBbTWDOlgmwjo8XXaN1GS6k,2848
 warn/scrapers/ne.py,sha256=JawuGJ3tCKvMd-N-p03gnltB4rol4QUJshMk2oyMPO4,4143
 warn/scrapers/nj.py,sha256=nwbMbeQuUJbYRVoyUyKZBmNqvqsXu3Habt-10r8DvZE,2230
 warn/scrapers/nm.py,sha256=55kDh65uOc-uMO8wuxPHhHN4Aw8Di2Fx66JBds8f2Io,3694
@@ -55,11 +56,9 @@ warn/scrapers/va.py,sha256=7Nle7qL0VNPiE653XyaP9HQqSfuJFDRr2kEkjOqLvFM,11269
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.169.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
-warn_scraper-1.2.169.dev0.dist-info/METADATA,sha256=idfAf3i-wQj6UbjYHXsG73B3pU8lvD4soFhnBis-e5c,1780
-warn_scraper-1.2.169.dev0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
-warn_scraper-1.2.169.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.169.dev0.dist-info/scm_file_list.json,sha256=PsUNADDLfiY6WCIaKYrGF-hSOzV6HMBxtHeJPnHAOAo,4147
-warn_scraper-1.2.169.dev0.dist-info/scm_version.json,sha256=4YY6UpShLyl1mHlnjJR2Fi1BirY30YZrm1c8Wg1OkHA,161
-warn_scraper-1.2.169.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
-warn_scraper-1.2.169.dev0.dist-info/RECORD,,
+warn_scraper-1.2.170.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
+warn_scraper-1.2.170.dev0.dist-info/METADATA,sha256=J76_LvKxe-qvqSaB0Hqkhi-7bEfvP6q5_kieFszDXxY,1780
+warn_scraper-1.2.170.dev0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+warn_scraper-1.2.170.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.170.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
+warn_scraper-1.2.170.dev0.dist-info/RECORD,,

warn_scraper-1.2.169.dev0.dist-info/scm_file_list.json

@@ -1,138 +0,0 @@
-{
-  "files": [
-    ".pre-commit-config.yaml",
-    "README.md",
-    "Makefile",
-    "setup.cfg",
-    "LICENSE",
-    "setup.py",
-    "pyproject.toml",
-    "Pipfile.lock",
-    "MANIFEST.in",
-    "Pipfile",
-    ".gitignore",
-    ".devcontainer/devcontainer.json",
-    "docs/reference.rst",
-    "docs/releasing.md",
-    "docs/Makefile",
-    "docs/sources.md",
-    "docs/contributing.rst",
-    "docs/index.rst",
-    "docs/requirements.txt",
-    "docs/make.bat",
-    "docs/usage.md",
-    "docs/conf.py",
-    "docs/scrapers/az.md",
-    "docs/scrapers/de.md",
-    "docs/scrapers/or.md",
-    "docs/scrapers/tx.md",
-    "docs/scrapers/sc.md",
-    "docs/scrapers/vt.md",
-    "docs/scrapers/job_center.md",
-    "docs/scrapers/dc.md",
-    "docs/scrapers/ut.md",
-    "docs/scrapers/mo.md",
-    "docs/scrapers/al.md",
-    "docs/scrapers/me.md",
-    "docs/scrapers/ks.md",
-    "docs/scrapers/in.md",
-    "docs/scrapers/ok.md",
-    "docs/scrapers/co.md",
-    "docs/scrapers/ia.md",
-    "docs/scrapers/va.md",
-    "docs/scrapers/md.md",
-    "docs/scrapers/ny.md",
-    "docs/scrapers/wi.md",
-    "docs/scrapers/ca.md",
-    "docs/_static/releasing-changelog-entered.png",
-    "docs/_static/releasing-publish-button.png",
-    "docs/_static/releasing-actions-start.png",
-    "docs/_static/gao-03-1003.pdf",
-    "docs/_static/releasing-tag-button.png",
-    "docs/_static/releasing-name-tag.png",
-    "docs/_static/releasing-draft-button.png",
-    "docs/_static/releasing-releases-button.png",
-    "docs/_static/R42693.pdf",
-    "docs/_static/releasing-actions-finished.png",
-    "docs/_static/releasing-pypi.png",
-    "docs/_static/releasing-name-release.png",
-    "docs/_static/releasing-release-published.png",
-    "docs/_static/releasing-changelog-button.png",
-    "docs/_templates/sources.md.tmpl",
-    "warn/__init__.py",
-    "warn/utils.py",
-    "warn/runner.py",
-    "warn/cache.py",
-    "warn/cli.py",
-    "warn/scrapers/az.py",
-    "warn/scrapers/la.py",
-    "warn/scrapers/tx.py",
-    "warn/scrapers/wa.py",
-    "warn/scrapers/wi.py",
-    "warn/scrapers/ga.py",
-    "warn/scrapers/co.py",
-    "warn/scrapers/de.py",
-    "warn/scrapers/__init__.py",
-    "warn/scrapers/ia.py",
-    "warn/scrapers/al.py",
-    "warn/scrapers/fl.py",
-    "warn/scrapers/nj.py",
-    "warn/scrapers/nd.py",
-    "warn/scrapers/mi.py",
-    "warn/scrapers/vt.py",
-    "warn/scrapers/mt.py",
-    "warn/scrapers/md.py",
-    "warn/scrapers/sc.py",
-    "warn/scrapers/ut.py",
-    "warn/scrapers/il.py",
-    "warn/scrapers/ky.py",
-    "warn/scrapers/or.py",
-    "warn/scrapers/in.py",
-    "warn/scrapers/ca.py",
-    "warn/scrapers/hi.py",
-    "warn/scrapers/ak.py",
-    "warn/scrapers/ri.py",
-    "warn/scrapers/ny.py",
-    "warn/scrapers/pa.py",
-    "warn/scrapers/dc.py",
-    "warn/scrapers/ct.py",
-    "warn/scrapers/mo.py",
-    "warn/scrapers/ok.py",
-    "warn/scrapers/tn.py",
-    "warn/scrapers/ks.py",
-    "warn/scrapers/oh.py",
-    "warn/scrapers/id.py",
-    "warn/scrapers/nm.py",
-    "warn/scrapers/me.py",
-    "warn/scrapers/va.py",
-    "warn/scrapers/ne.py",
-    "warn/scrapers/sd.py",
-    "warn/scrapers/ms.py",
-    "warn/platforms/__init__.py",
-    "warn/platforms/job_center/__init__.py",
-    "warn/platforms/job_center/utils.py",
-    "warn/platforms/job_center/cache.py",
-    "warn/platforms/job_center/site.py",
-    "warn/platforms/job_center/urls.py",
-    "warn/pdfrodent/__init__.py",
-    "warn/pdfrodent/pdfrodent.py",
-    "tests/__init__.py",
-    "tests/test_job_center.py",
-    "tests/test_job_center_cache.py",
-    "tests/test_openpyxl.py",
-    "tests/conftest.py",
-    "tests/test_cache.py",
-    "tests/test_delete.py",
-    "tests/fixtures/2021_page_2.html",
-    "tests/fixtures/2021_page_1.html",
-    "tests/cassettes/test_scrape_integration.yaml",
-    "tests/cassettes/test_cached_search_results.yaml",
-    "tests/cassettes/test_no_results.yaml",
-    "tests/cassettes/test_paged_results.yaml",
-    "tests/cassettes/test_missing_detail_page_values.yaml",
-    "tests/cassettes/test_cached_detail_pages.yaml",
-    ".github/dependabot-disabled.yml",
-    ".github/workflows/continuous-deployment.yml",
-    ".github/workflows/continuous-deployment.yml.broken-tests"
-  ]
-}

warn_scraper-1.2.169.dev0.dist-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "1.2.168",
-  "distance": 0,
-  "node": "gf7b3dd26af1f3ad700762504bd7c5c0d23979507",
-  "dirty": true,
-  "branch": "HEAD",
-  "node_date": "2026-07-14"
-}