warn-scraper 1.2.80__py3-none-any.whl → 1.2.82__py3-none-any.whl

warn/scrapers/md.py

@@ -1,6 +1,7 @@
 import logging
 import re
 from pathlib import Path
+from time import sleep
 
 from bs4 import BeautifulSoup
 
@@ -16,6 +17,8 @@ __source__ = {
 
 logger = logging.getLogger(__name__)
 
+naptime = 3
+
 
 def scrape(
     data_dir: Path = utils.WARN_DATA_DIR,
@@ -42,6 +45,8 @@ def scrape(
     # Save it to the cache
     cache.write("md/source.html", html)
 
+    sleep(naptime)  # Try to stop blocked connections by being less aggressive
+
     # Parse the list of links
     soup = BeautifulSoup(html, "html.parser")
     a_list = soup.find_all("a", {"class": "sub"})
@@ -61,6 +66,8 @@ def scrape(
         # Save it to the cache
         cache.write(f"md/{href}.html", html)
 
+        sleep(naptime)  # Try to stop blocked connections by being less aggressive
+
         # Add it to the list
         html_list.append(html)
 

warn/scrapers/ri.py

@@ -61,6 +61,7 @@ def scrape(
                 dirty_list.extend(localrows)
 
             headers = dirty_list[1]  # Skip false header at position 0
+            headers = [x for x in headers if x is not None]
             headers[2] = (
                 headers[2]
                 .replace("Company Name ", "Company Name")
@@ -85,9 +86,14 @@ def scrape(
                         logger.debug(f"Got : {row}")
                     else:
                         line = {}
-                        for i, fieldname in enumerate(headers):
-                            line[fieldname] = row[i]
-                        row_list.append(line)
+                        if len(headers) > len(row):
+                            logger.debug(
+                                f"{len(row)} items found, vs. expected {len(headers)}. Dropping row: {row}"
+                            )
+                        else:
+                            for i, fieldname in enumerate(headers):
+                                line[fieldname] = row[i]
+                            row_list.append(line)
             # dirty_list = None
             logger.debug(
                 f"Successfully merged {len(row_list)-1:,} records from new spreadsheet."

{warn_scraper-1.2.80.dist-info → warn_scraper-1.2.82.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.80
+Version: 1.2.82
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.80.dist-info → warn_scraper-1.2.82.dist-info}/RECORD

@@ -43,7 +43,7 @@ warn/scrapers/in.py,sha256=dAT40ROhhKiwLcwa_YJ6EyhsYBLe0IX2rOWXmNa6JMs,2026
 warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
 warn/scrapers/ky.py,sha256=XjIojMpaoKbypa7l23IybP02jBijBCJG5UGqfO-EYjg,4365
 warn/scrapers/la.py,sha256=60z-4LZY5xp6aX8r6HGGW3FaOVEGnxlG2Mfgpt4G2WE,12877
-warn/scrapers/md.py,sha256=PHvL81V0AwAhYhTTEmXBhbfZMBrBJipvUwXKSpizifA,2845
+warn/scrapers/md.py,sha256=e-tiiKwr9dNEemtk7SWMv317Nv-qEDf5xPNcMU8AZDQ,3045
 warn/scrapers/me.py,sha256=q36F4yJ7hvZsLayA3uBS1romo4X3Qf-sEi2Y7LAQCi8,1172
 warn/scrapers/mi.py,sha256=9clZ9mATEJwdVLzDo_h66rK0aV5Zc7GGQ7AauutS6Wo,3591
 warn/scrapers/mo.py,sha256=wnnwQAiVPwuheMqptMXZpyQdiKNghhKwTO-Bnh9oXoU,3492
@@ -55,7 +55,7 @@ warn/scrapers/ny.py,sha256=kuBdgF1C_GnHEWlaLAas6zJG2Xhfj3c4q_-tuJLX2rk,3615
 warn/scrapers/oh.py,sha256=2MEB_0AT37dsAsrhdl_Y0LUNHu0xGy4B1F7aSMhuUu0,3151
 warn/scrapers/ok.py,sha256=qJE49VY6dMhbokFB9IAOL2XyuYSJpEKKxITPO9sUHS4,1197
 warn/scrapers/or.py,sha256=0PjyrW3CHdxtHhqEo3Ob-9B6YckACoBD3K0c4FPQUcg,5208
-warn/scrapers/ri.py,sha256=vBbXFP5ClvqlOc_srR8sHsA8lpi7eLuMYm7ydUY5Fxo,4163
+warn/scrapers/ri.py,sha256=EUyLy59eNiYHqiJR8C0YcJrZtp09KyVc45AFD0_Uc0U,4497
 warn/scrapers/sc.py,sha256=p3kscSNSW9C8C5QaSUbCAo6XibgB7G2iH6zaMH7Mnsc,4819
 warn/scrapers/sd.py,sha256=_4R19Ybzsyx1PvcWV3_laJmJ3etrwVGfhNEQm6njwoA,1904
 warn/scrapers/tn.py,sha256=i1H7c09Ea3CDrTXqqRMLBMPT_34QtGA0-x7T8rm_j5Q,2945
@@ -65,9 +65,9 @@ warn/scrapers/va.py,sha256=13lhkQrSkPGHEiWUuf1qiS890PWYE5gV-TgISpoiQnc,1711
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.80.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warn_scraper-1.2.80.dist-info/METADATA,sha256=R3hOSq76-i_I4IBWW1NX1cGg7SoJM8PNh2dBMefHqUs,2025
-warn_scraper-1.2.80.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-warn_scraper-1.2.80.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.80.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
-warn_scraper-1.2.80.dist-info/RECORD,,
+warn_scraper-1.2.82.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warn_scraper-1.2.82.dist-info/METADATA,sha256=kxvv_1p1AQwjc40wHLkQddflASZc9I4_GxtcySjdNIo,2025
+warn_scraper-1.2.82.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+warn_scraper-1.2.82.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.82.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
+warn_scraper-1.2.82.dist-info/RECORD,,