warn-scraper 1.2.111__py3-none-any.whl → 1.2.112__py3-none-any.whl

warn/scrapers/la.py

@@ -1,16 +1,19 @@
 import logging
 import os
 import re
+import sys
+from base64 import b64decode
 from datetime import datetime
 from pathlib import Path
 
 import pdfplumber
+import requests
 from bs4 import BeautifulSoup
 
 from .. import utils
 from ..cache import Cache
 
-__authors__ = ["chriszs"]
+__authors__ = ["chriszs", "stucka"]
 __tags__ = ["html", "pdf"]
 __source__ = {
     "name": "Louisiana Workforce Commission",
@@ -33,6 +36,14 @@ def scrape(
 
     Returns: the Path where the file is written
     """
+    try:
+        zyte_api_key = os.environ["ZYTE_API_KEY"]
+    except KeyError:
+        logger.error(
+            "No ZYTE_API_KEY variable found in environment. Please get an API key from Zyte and export it."
+        )
+        sys.exit(1)
+
     # Fire up the cache
     cache = Cache(cache_dir)
 
@@ -43,10 +54,22 @@ def scrape(
 
     # Download the root page
     url = f"{base_url}Downloads/{file_base}.asp"
-    html = utils.get_url(url).text
+    api_response = requests.post(
+        "https://api.zyte.com/v1/extract",
+        auth=(zyte_api_key, ""),
+        json={
+            "url": url,
+            "httpResponseBody": True,
+            "followRedirect": True,
+        },
+    )
+    html_bytes: bytes = b64decode(api_response.json()["httpResponseBody"])
+    # html = utils.get_url(url).text
+    html = html_bytes.decode("utf-8", errors="backslashreplace")
 
     # Save it to the cache
-    cache_key = f"{state_code}/{file_base}.html"
+    cache_key = cache_dir / f"{state_code}/{file_base}.html"
+    utils.create_directory(Path(cache_key), is_file=True)
     cache.write(cache_key, html)
 
     # Parse out the links to WARN notice PDFs
@@ -59,9 +82,28 @@ def scrape(
         if "WARN Notices" in link.text:
             # Download the PDF
             pdf_url = f"{base_url}{link['href']}"
-            pdf_path = _read_or_download(cache, state_code, pdf_url)
+            logger.debug(pdf_url)
+            api_response = requests.post(
+                "https://api.zyte.com/v1/extract",
+                auth=(zyte_api_key, ""),
+                json={
+                    "url": pdf_url,
+                    "httpResponseBody": True,
+                    "followRedirect": True,
+                },
+            )
+            http_response_body: bytes = b64decode(
+                api_response.json()["httpResponseBody"]
+            )
+            pdf_path = cache_dir / f"{state_code}/{os.path.basename(pdf_url)}"
+
+            with open(pdf_path, "wb") as fp:
+                fp.write(http_response_body)
+
+            # pdf_path = _read_or_download(cache, state_code, pdf_url)
 
             # Process the PDF
+            logger.debug(f"Attempting to parse {pdf_path}")
             rows = _process_pdf(pdf_path)
             all_rows.extend(rows)
 

{warn_scraper-1.2.111.dist-info → warn_scraper-1.2.112.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.111
+Version: 1.2.112
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.111.dist-info → warn_scraper-1.2.112.dist-info}/RECORD

@@ -42,7 +42,7 @@ warn/scrapers/il.py,sha256=sygdvsNuB_Gvu3o_HidtpSP4FLz0szKb1zEHqGxVtlI,1563
 warn/scrapers/in.py,sha256=dAT40ROhhKiwLcwa_YJ6EyhsYBLe0IX2rOWXmNa6JMs,2026
 warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
 warn/scrapers/ky.py,sha256=XjIojMpaoKbypa7l23IybP02jBijBCJG5UGqfO-EYjg,4365
-warn/scrapers/la.py,sha256=60z-4LZY5xp6aX8r6HGGW3FaOVEGnxlG2Mfgpt4G2WE,12877
+warn/scrapers/la.py,sha256=z2dCJ0obnKy47ZL21MrtKTo5AdexRDj0BBxyV2uIY_8,14340
 warn/scrapers/md.py,sha256=hwgxXQnhyBWm8qF1dvxIThAX1MkrZbXLwRI9inO5t8g,4060
 warn/scrapers/me.py,sha256=q36F4yJ7hvZsLayA3uBS1romo4X3Qf-sEi2Y7LAQCi8,1172
 warn/scrapers/mi.py,sha256=9clZ9mATEJwdVLzDo_h66rK0aV5Zc7GGQ7AauutS6Wo,3591
@@ -65,9 +65,9 @@ warn/scrapers/va.py,sha256=7Nle7qL0VNPiE653XyaP9HQqSfuJFDRr2kEkjOqLvFM,11269
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.111.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
-warn_scraper-1.2.111.dist-info/METADATA,sha256=DW2-aOg2XJoXTqAN9fgs-tkMf7JjoijXxr93sUpEVfA,2385
-warn_scraper-1.2.111.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-warn_scraper-1.2.111.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.111.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
-warn_scraper-1.2.111.dist-info/RECORD,,
+warn_scraper-1.2.112.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
+warn_scraper-1.2.112.dist-info/METADATA,sha256=dHhKPqTbOMGVMZ_eDRMnQ1TEjaoBPvzdqHGPwyWDXsU,2385
+warn_scraper-1.2.112.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+warn_scraper-1.2.112.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.112.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
+warn_scraper-1.2.112.dist-info/RECORD,,