warn-scraper 1.2.80__tar.gz → 1.2.82__tar.gz

{warn_scraper-1.2.80/warn_scraper.egg-info → warn_scraper-1.2.82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.80
+Version: 1.2.82
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.80 → warn_scraper-1.2.82}/warn/scrapers/md.py

@@ -1,6 +1,7 @@
 import logging
 import re
 from pathlib import Path
+from time import sleep
 
 from bs4 import BeautifulSoup
 
@@ -16,6 +17,8 @@ __source__ = {
 
 logger = logging.getLogger(__name__)
 
+naptime = 3
+
 
 def scrape(
     data_dir: Path = utils.WARN_DATA_DIR,
@@ -42,6 +45,8 @@ def scrape(
     # Save it to the cache
     cache.write("md/source.html", html)
 
+    sleep(naptime)  # Try to stop blocked connections by being less aggressive
+
     # Parse the list of links
     soup = BeautifulSoup(html, "html.parser")
     a_list = soup.find_all("a", {"class": "sub"})
@@ -61,6 +66,8 @@ def scrape(
         # Save it to the cache
         cache.write(f"md/{href}.html", html)
 
+        sleep(naptime)  # Try to stop blocked connections by being less aggressive
+
         # Add it to the list
         html_list.append(html)
 

{warn_scraper-1.2.80 → warn_scraper-1.2.82}/warn/scrapers/ri.py

@@ -61,6 +61,7 @@ def scrape(
                 dirty_list.extend(localrows)
 
             headers = dirty_list[1]  # Skip false header at position 0
+            headers = [x for x in headers if x is not None]
             headers[2] = (
                 headers[2]
                 .replace("Company Name ", "Company Name")
@@ -85,9 +86,14 @@ def scrape(
                         logger.debug(f"Got : {row}")
                     else:
                         line = {}
-                        for i, fieldname in enumerate(headers):
-                            line[fieldname] = row[i]
-                        row_list.append(line)
+                        if len(headers) > len(row):
+                            logger.debug(
+                                f"{len(row)} items found, vs. expected {len(headers)}. Dropping row: {row}"
+                            )
+                        else:
+                            for i, fieldname in enumerate(headers):
+                                line[fieldname] = row[i]
+                            row_list.append(line)
             # dirty_list = None
             logger.debug(
                 f"Successfully merged {len(row_list)-1:,} records from new spreadsheet."

{warn_scraper-1.2.80 → warn_scraper-1.2.82/warn_scraper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.80
+Version: 1.2.82
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News