warn-scraper 1.2.110__tar.gz → 1.2.112__tar.gz

{warn_scraper-1.2.110/warn_scraper.egg-info → warn_scraper-1.2.112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.110
+Version: 1.2.112
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.110 → warn_scraper-1.2.112}/warn/scrapers/ia.py

@@ -58,6 +58,8 @@ def scrape(
 
     # Parse it, minus the header
     row_list += utils.parse_excel(historic_excel_path, keep_header=False)
+    if "Iowa WARN Log" in row_list[0]:
+        del row_list[0]
 
     # Set the export path
     data_path = data_dir / "ia.csv"

{warn_scraper-1.2.110 → warn_scraper-1.2.112}/warn/scrapers/la.py

@@ -1,16 +1,19 @@
 import logging
 import os
 import re
+import sys
+from base64 import b64decode
 from datetime import datetime
 from pathlib import Path
 
 import pdfplumber
+import requests
 from bs4 import BeautifulSoup
 
 from .. import utils
 from ..cache import Cache
 
-__authors__ = ["chriszs"]
+__authors__ = ["chriszs", "stucka"]
 __tags__ = ["html", "pdf"]
 __source__ = {
     "name": "Louisiana Workforce Commission",
@@ -33,6 +36,14 @@ def scrape(
 
     Returns: the Path where the file is written
     """
+    try:
+        zyte_api_key = os.environ["ZYTE_API_KEY"]
+    except KeyError:
+        logger.error(
+            "No ZYTE_API_KEY variable found in environment. Please get an API key from Zyte and export it."
+        )
+        sys.exit(1)
+
     # Fire up the cache
     cache = Cache(cache_dir)
 
@@ -43,10 +54,22 @@ def scrape(
 
     # Download the root page
     url = f"{base_url}Downloads/{file_base}.asp"
-    html = utils.get_url(url).text
+    api_response = requests.post(
+        "https://api.zyte.com/v1/extract",
+        auth=(zyte_api_key, ""),
+        json={
+            "url": url,
+            "httpResponseBody": True,
+            "followRedirect": True,
+        },
+    )
+    html_bytes: bytes = b64decode(api_response.json()["httpResponseBody"])
+    # html = utils.get_url(url).text
+    html = html_bytes.decode("utf-8", errors="backslashreplace")
 
     # Save it to the cache
-    cache_key = f"{state_code}/{file_base}.html"
+    cache_key = cache_dir / f"{state_code}/{file_base}.html"
+    utils.create_directory(Path(cache_key), is_file=True)
     cache.write(cache_key, html)
 
     # Parse out the links to WARN notice PDFs
@@ -59,9 +82,28 @@ def scrape(
         if "WARN Notices" in link.text:
             # Download the PDF
             pdf_url = f"{base_url}{link['href']}"
-            pdf_path = _read_or_download(cache, state_code, pdf_url)
+            logger.debug(pdf_url)
+            api_response = requests.post(
+                "https://api.zyte.com/v1/extract",
+                auth=(zyte_api_key, ""),
+                json={
+                    "url": pdf_url,
+                    "httpResponseBody": True,
+                    "followRedirect": True,
+                },
+            )
+            http_response_body: bytes = b64decode(
+                api_response.json()["httpResponseBody"]
+            )
+            pdf_path = cache_dir / f"{state_code}/{os.path.basename(pdf_url)}"
+
+            with open(pdf_path, "wb") as fp:
+                fp.write(http_response_body)
+
+            # pdf_path = _read_or_download(cache, state_code, pdf_url)
 
             # Process the PDF
+            logger.debug(f"Attempting to parse {pdf_path}")
             rows = _process_pdf(pdf_path)
             all_rows.extend(rows)
 

{warn_scraper-1.2.110 → warn_scraper-1.2.112/warn_scraper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.110
+Version: 1.2.112
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News