warn-scraper 1.2.111__tar.gz → 1.2.113__tar.gz

{warn_scraper-1.2.111/warn_scraper.egg-info → warn_scraper-1.2.113}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.111
+Version: 1.2.113
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.111 → warn_scraper-1.2.113}/warn/scrapers/la.py

@@ -10,7 +10,7 @@ from bs4 import BeautifulSoup
 from .. import utils
 from ..cache import Cache
 
-__authors__ = ["chriszs"]
+__authors__ = ["chriszs", "stucka"]
 __tags__ = ["html", "pdf"]
 __source__ = {
     "name": "Louisiana Workforce Commission",
@@ -43,10 +43,11 @@ def scrape(
 
     # Download the root page
     url = f"{base_url}Downloads/{file_base}.asp"
-    html = utils.get_url(url).text
+    htmlbin, html = utils.get_with_zyte(url)
 
     # Save it to the cache
-    cache_key = f"{state_code}/{file_base}.html"
+    cache_key = cache_dir / f"{state_code}/{file_base}.html"
+    utils.create_directory(Path(cache_key), is_file=True)
     cache.write(cache_key, html)
 
     # Parse out the links to WARN notice PDFs
@@ -59,9 +60,14 @@ def scrape(
         if "WARN Notices" in link.text:
             # Download the PDF
             pdf_url = f"{base_url}{link['href']}"
-            pdf_path = _read_or_download(cache, state_code, pdf_url)
+            rawbin, rawtext = utils.get_with_zyte(pdf_url)
+            pdf_path = cache_dir / f"{state_code}/{os.path.basename(pdf_url)}"
+
+            with open(pdf_path, "wb") as fp:
+                fp.write(rawbin)
 
             # Process the PDF
+            logger.debug(f"Attempting to parse {pdf_path}")
             rows = _process_pdf(pdf_path)
             all_rows.extend(rows)
 

{warn_scraper-1.2.111 → warn_scraper-1.2.113}/warn/utils.py

@@ -1,7 +1,9 @@
 import csv
+import json
 import logging
 import os
 import typing
+from base64 import b64decode, b64encode
 from pathlib import Path
 from time import sleep
 
@@ -94,6 +96,103 @@ def save_if_good_url(filename, url, **kwargs):
     return success_flag, content
 
 
+def get_with_zyte(url):
+    """Use Zyte as a proxy server to retrieve data not available without it.
+
+    Args:
+        url (str): URL to retrieve
+    Returns:
+        returnbin (bin): raw binary representation of returned data object
+        returntext (str): utf-8 conversion of returned data object, e.g., HTML
+    Failures:
+        Returns (None, None) if it encounters a problem and logs an error.
+    Requires:
+        ZYTE_API_KEY to be set in environment
+    """
+    logger.debug(f"Seeking to fetch {url} with Zyte")
+    try:
+        zyte_api_key = os.environ["ZYTE_API_KEY"]
+    except KeyError:
+        logger.error(
+            "No ZYTE_API_KEY variable found in environment. Please get an API key from Zyte and export it."
+        )
+        return (None, None)
+
+    api_response = requests.post(
+        "https://api.zyte.com/v1/extract",
+        auth=(zyte_api_key, ""),
+        json={
+            "url": url,
+            "httpResponseBody": True,
+            "followRedirect": True,
+        },
+    )
+
+    if not api_response.ok:
+        logger.error(
+            f"Error downloading {url} with get_with_zyte. Repsonse code: {api_response.status_code}"
+        )
+        return (None, None)
+    returnbin: bytes = b64decode(api_response.json()["httpResponseBody"])
+    returntext: str = returnbin.decode("utf-8", errors="backslashreplace")
+    logger.debug(f"Fetched {url}")
+    return (returnbin, returntext)
+
+
+def post_with_zyte(url, payload):
+    """Use Zyte as a proxy server to retrieve data not available without it.
+
+    Args:
+        url (str): URL to retrieve
+        payload: (dict, str or binary): POST body.
+            If type dict: Convert to utf-8 text then:
+            If type str: Convert to b64encoded
+    Returns:
+        returnbin (bin): raw binary representation of returned data object
+        returntext (str): utf-8 conversion of returned data object, e.g., HTML
+    Failures:
+        Returns (None, None) if it encounters a problem and logs an error.
+    Requires:
+        ZYTE_API_KEY to be set in environment
+    """
+    logger.debug(f"Seeking to fetch {url} with Zyte")
+    try:
+        zyte_api_key = os.environ["ZYTE_API_KEY"]
+    except KeyError:
+        logger.error(
+            "No ZYTE_API_KEY variable found in environment. Please get an API key from Zyte and export it."
+        )
+        return (None, None)
+
+    if isinstance(payload, dict):
+        payload = json.dumps(payload)
+
+    if isinstance(payload, str):
+        payload = b64encode(payload.encode("utf-8"))
+
+    api_response = requests.post(
+        "https://api.zyte.com/v1/extract",
+        auth=(zyte_api_key, ""),
+        json={
+            "url": url,
+            "httpRequestMethod": "POST",
+            "httpRequestBody": payload,
+            "httpResponseBody": True,
+            "followRedirect": True,
+        },
+    )
+
+    if not api_response.ok:
+        logger.error(
+            f"Error downloading {url} with post_with_zyte. Repsonse code: {api_response.status_code}. Reponse: {api_response.json()}"
+        )
+        return (None, None)
+    returnbin: bytes = b64decode(api_response.json()["httpResponseBody"])
+    returntext: str = returnbin.decode("utf-8", errors="backslashreplace")
+    logger.debug(f"Fetched {url}")
+    return (returnbin, returntext)
+
+
 def write_rows_to_csv(output_path: Path, rows: list, mode="w"):
     """Write the provided list to the provided path as comma-separated values.
 
@@ -109,9 +208,7 @@ def write_rows_to_csv(output_path: Path, rows: list, mode="w"):
         writer.writerows(rows)
 
 
-def write_dict_rows_to_csv(
-    output_path, headers, rows, mode="w", extrasaction="raise", encoding="utf-8"
-):
+def write_dict_rows_to_csv(output_path, headers, rows, mode="w", extrasaction="raise"):
     """Write the provided dictionary to the provided path as comma-separated values.
 
     Args:
@@ -123,7 +220,7 @@ def write_dict_rows_to_csv(
     """
     create_directory(output_path, is_file=True)
     logger.debug(f"Writing {len(rows)} rows to {output_path}")
-    with open(output_path, mode, newline="", encoding=encoding) as f:
+    with open(output_path, mode, newline="") as f:
         # Create the writer object
         writer = csv.DictWriter(f, fieldnames=headers, extrasaction=extrasaction)
         # If we are writing a new row ...

{warn_scraper-1.2.111 → warn_scraper-1.2.113/warn_scraper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.111
+Version: 1.2.113
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News